NimYAML/src/yaml/sequential.nim

import streams, tables, strutils

import "private/lexer"

type
    YamlParserEventKind* = enum
        yamlStartDocument, yamlEndDocument, yamlStartMap, yamlEndMap,
        yamlStartSequence, yamlEndSequence, yamlScalar, yamlAlias,
        yamlError, yamlWarning
    
    YamlParserEvent* = ref object
        case kind*: YamlParserEventKind
        of yamlStartMap, yamlStartSequence:
            objAnchor* : string # may be nil, may not be empty
            objTag*    : string # may not be nil or empty, is a complete URI.
        of yamlScalar:
            scalarAnchor* : string # may be nil
            scalarTag*    : string # may not be nil, is a complete URI.
            scalarContent*: string # may not be nil (but empty)
        of yamlEndMap, yamlEndSequence, yamlStartDocument, yamlEndDocument:
            discard
        of yamlAlias:
            aliasName*   : string # may not be nil nor empty
        of yamlError, yamlWarning:
            description* : string
            line*        : int
            column*      : int
    
    YamlParserState = enum
        ylInitial, ylSkipDirective, ylBlockLineStart, ylBlockAfterScalar,
        ylBlockAfterColon, ylBlockLineEnd, ylFlow
    
    OutcomeEnum = enum
        oOkay, oWarn, oContinue
    
    LevelKind = enum
        lUnknown, lSequence, lMap
    
    DocumentLevel = object
        kind: LevelKind
        indicatorColumn: int
        readKey: bool
        anchor: string
        tag: string

proc `==`*(left: YamlParserEvent, right: YamlParserEvent): bool =
    if left.kind != right.kind:
        return false
    case left.kind
    of yamlStartDocument, yamlEndDocument, yamlEndMap, yamlEndSequence:
        result = true
    of yamlStartMap, yamlStartSequence:
        result = left.objAnchor == right.objAnchor and
                 left.objTag == right.objTag
    of yamlScalar:
        result = left.scalarAnchor == right.scalarAnchor and
                 left.scalarTag == right.scalarTag and
                 left.scalarContent == right.scalarContent
    of yamlAlias:
        result = left.aliasName == right.aliasName
    of yamlError, yamlWarning:
        result = left.description == right.description and
                 left.line == right.line and left.column == right.column
    
template yieldWarning(d: string) {.dirty.} =
    yield YamlParserEvent(kind: yamlWarning, description: d,
                          line: lex.line, column: lex.column)

template yieldError(d: string) {.dirty.} =
    yield YamlParserEvent(kind: yamlError, description: d,
                          line: lex.line, column: lex.column)

template closeLevel() {.dirty.} =
    case level.kind
    of lUnknown:
        yield YamlParserEvent(kind: yamlScalar, scalarAnchor: level.anchor,
                              scalarTag: level.tag, scalarContent: "")
    of lSequence:
        yield YamlParserEvent(kind: yamlEndSequence)
    of lMap:
        yield YamlParserEvent(kind: yamlEndMap)

template closeLevelsByIndicator() {.dirty.} =
    while levels.len > 0:
        let level = levels[levels.high]
        if level.indicatorColumn > lex.column:
            closeLevel()
        elif level.indicatorColumn == -1:
            if levels[levels.high - 1].indicatorColumn >= lex.column:
                closeLevel()
            else:
                break
        else:
            break
        discard levels.pop()

template closeAllLevels() {.dirty.} =
    while levels.len > 0:
        var level = levels.pop()
        closeLevel()

iterator events*(input: Stream): YamlParserEvent {.closure.} =
    var
        state = ylInitial
        lex   : YamlLexer
        foundYamlDirective = false
        tagShorthands = initTable[string, string]()
        levels = newSeq[DocumentLevel]()
        curIndentation: int
        cachedScalar: YamlParserEvent
        cachedScalarIndentation: int
    lex.open(input)
    
    var nextToken = tokens
    var token = nextToken(lex)
    while not finished(nextToken):
        case state
        of ylInitial:
            case token.kind
            of yamlYamlDirective:
                if foundYamlDirective:
                    yield YamlParserEvent(kind: yamlError,
                            description: "Duplicate %YAML tag",
                            line:   lex.line,
                            column: lex.column)
                    state = ylSkipDirective
                else:
                    var
                        outcome = oOkay
                        actualVersion = ""
                    for version in [1, 2]:
                        token = nextToken(lex)
                        if finished(nextToken):
                            yieldError("Missing or badly formatted YAML version")
                            outcome = oContinue
                            break
                        if token.kind != yamlVersionPart:
                            yieldError("Missing or badly formatted YAML version")
                            outcome = oContinue
                            break
                        if parseInt(lex.content) != version:
                            outcome = oWarn
                        if actualVersion.len > 0: actualVersion &= "."
                        actualVersion &= $version
                    case outcome
                    of oContinue:
                        continue
                    of oWarn:
                        yieldWarning("Unsupported version: " & actualVersion &
                                     ", trying to parse anyway")
                    else:
                        discard
                    foundYamlDirective = true
            of yamlTagDirective:
                token = nextToken(lex)
                if finished(nextToken):
                    yieldError("Incomplete %TAG directive")
                    continue
                if token.kind != yamlTagHandle:
                    yieldError("Invalid token (expected tag handle)")
                    state = ylSkipDirective
                    continue
                let tagHandle = lex.content
                token = nextToken(lex)
                if finished(nextToken):
                    yieldError("Incomplete %TAG directive")
                    continue
                if token.kind != yamlTagURI:
                    yieldError("Invalid token (expected tag URI)")
                    state = ylSkipDirective
                    continue
                tagShorthands[tagHandle] = lex.content
            of yamlUnknownDirective:
                yieldWarning("Unknown directive: " & lex.content)
                state = ylSkipDirective
            of yamlComment:
                discard
            of yamlDirectivesEnd:
                yield YamlParserEvent(kind: yamlStartDocument)
                state = ylBlockLinestart
            of yamlDocumentEnd, yamlStreamEnd:
                yield YamlParserEvent(kind: yamlStartDocument)
                yield YamlParserEvent(kind: yamlEndDocument)
            else:
                yield YamlParserEvent(kind: yamlStartDocument)
                state = ylBlockLineStart
                continue
        of ylSkipDirective:
            if token.kind notin [yamlUnknownDirectiveParam, yamlTagHandle,
                                 yamlTagURI, yamlVersionPart, yamlComment]:
                state = ylInitial
                continue
        of ylBlockLineStart:
            case token.kind
            of yamlLineStart:
                discard
            of yamlDash:
                closeLevelsByIndicator()
                if levels.len > 0:
                    var level = addr(levels[levels.high])
                    if level.kind == lUnknown:
                        level.kind = lSequence
                        level.indicatorColumn = lex.column
                        levels.add(DocumentLevel(kind: lUnknown,
                                                 indicatorColumn: -1,
                                                 readKey: false,
                                                 anchor: nil, tag: nil))
                        yield YamlParserEvent(kind: yamlStartSequence,
                                              objAnchor: level.anchor,
                                              objTag: level.tag)
                    elif level.indicatorColumn < lex.column:
                        yieldError("Invalid indentation for '-'")
                    elif level.kind == lSequence:
                        levels.add(DocumentLevel(kind: lUnknown,
                                                 indicatorColumn: -1,
                                                 readKey: false,
                                                 anchor: nil, tag: nil))
                    else:
                        yieldError("Unexpected token: '-'")
                else:
                    levels.add(DocumentLevel(kind: lSequence,
                                             indicatorColumn: lex.column,
                                             readKey: false,
                                             anchor: nil, tag: nil))
                    levels.add(DocumentLevel(kind: lUnknown,
                                             indicatorColumn: -1,
                                             readKey: false,
                                             anchor: nil, tag: nil))
                    yield YamlParserEvent(kind: yamlStartSequence,
                                          objAnchor: nil, objTag: nil)
            of yamlQuestionmark, yamlColon:
                closeLevelsByIndicator()
                if levels.len > 0:
                    var level = addr(levels[levels.high])
                    if level.kind == lUnknown:
                        level.kind = lMap
                        level.indicatorColumn = lex.column
                        levels.add(DocumentLevel(kind: lUnknown,
                                                 indicatorColumn: -1,
                                                 readKey: true,
                                                 anchor: nil, tag: nil))
                        yield YamlParserEvent(kind: yamlStartMap,
                                              objAnchor: level.anchor,
                                              objTag: level.tag)
                        if token.kind == yamlColon:
                            yield YamlParserEvent(kind: yamlScalar,
                                                  scalarAnchor: level.anchor,
                                                  scalarTag: level.tag,
                                                  scalarContent: "")
                            level.readKey = false
                    elif level.indicatorColumn < lex.column:
                        yieldError("Invalid indentation for '?'")
                    elif level.kind == lMap and level.readKey ==
                            (token.kind == yamlColon):
                        level.readKey = true
                        levels.add(DocumentLevel(kind: lUnknown,
                                                 indicatorColumn: -1,
                                                 readKey: (token.kind == yamlQuestionmark),
                                                 anchor: nil, tag: nil))
                    else:
                        yieldError("Unexpected token: '?'")
                else:
                    levels.add(DocumentLevel(kind: lMap,
                                             indicatorColumn: lex.column,
                                             readKey: true,
                                             anchor: nil, tag: nil))
                    var level = addr(levels[levels.high])
                    levels.add(DocumentLevel(kind: lUnknown,
                                             indicatorColumn: -1,
                                             readKey: false,
                                             anchor: nil, tag: nil))
                    yield YamlParserEvent(kind: yamlStartMap,
                                          objAnchor: nil,
                                          objTag: nil)
                    if token.kind == yamlColon:
                        yield YamlParserEvent(kind: yamlScalar,
                                              scalarAnchor: nil,
                                              scalarTag: nil,
                                              scalarContent: "")
                        level.readKey = false
            of yamlTagHandle:
                var level = addr(levels[levels.high])
                let handle = lex.content
                if tagShorthands.hasKey(handle):
                    token = nextToken(lex)
                    if finished(nextToken):
                        yieldError("Missing tag suffix")
                        continue
                    if token.kind != yamlTagSuffix:
                        yieldError("Missing tag suffix")
                        continue
                    level.tag = tagShorthands[handle] & lex.content
                else:
                    yieldError("Unknown tag shorthand: " & handle)
            of yamlVerbatimTag:
                levels[levels.high].tag = lex.content
            of lexer.yamlScalar:
                closeLevelsByIndicator()
                if levels.len > 0:
                    let level = levels.pop()
                    if level.kind != lUnknown:
                        yieldError("Unexpected scalar in " & $level.kind)
                    else:
                        cachedScalar = YamlParserEvent(kind: yamlScalar,
                                scalarAnchor: level.anchor,
                                scalarTag: level.tag,
                                scalarContent: lex.content)
                        cachedScalarIndentation = lex.column
                else:
                    cachedScalar = YamlParserEvent(kind: yamlScalar,
                            scalarAnchor: nil, scalarTag: nil,
                            scalarContent: lex.content)
                state = ylBlockAfterScalar
            of yamlStreamEnd:
                closeAllLevels()
                yield YamlParserEvent(kind: yamlEndDocument)
                break
            of yamlDocumentEnd:
                closeAllLevels()
                yield YamlParserEvent(kind: yamlEndDocument)
                state = ylInitial
            else:
                yieldError("Unexpected token: " & $token.kind)
        of ylBlockAfterScalar:
            case token.kind
            of yamlColon:
                var level: ptr DocumentLevel = nil
                if levels.len > 0:
                    level = addr(levels[levels.high])
                if level == nil or level.kind != lUnknown:
                    levels.add(DocumentLevel(kind: lUnknown))
                    level = addr(levels[levels.high])
                level.kind = lMap
                level.indicatorColumn = lex.column
                level.readKey = true
                yield YamlParserEvent(kind: yamlStartMap)
                yield cachedScalar
                levels.add(DocumentLevel(kind: lUnknown,
                                         indicatorColumn: -1))
                cachedScalar = nil
                state = ylBlockAfterColon
            of yamlLineStart:
                state = ylBlockLineStart
            of yamlStreamEnd:
                yield cachedScalar
                closeAllLevels()
                yield YamlParserEvent(kind: yamlEndDocument)
                break
            else:
                yieldError("Unexpected token: " & $token.kind)
        of ylBlockAfterColon:
            case token.kind
            of lexer.yamlScalar:
                var level = levels.pop()
                yield YamlParserEvent(kind: yamlScalar,
                        scalarAnchor: level.anchor, scalarTag: level.tag,
                        scalarContent: lex.content)
                state = ylBlockLineEnd
            of yamlLineStart:
                state = ylBlockLineStart
            of yamlStreamEnd:
                closeAllLevels()
                yield YamlParserEvent(kind: yamlEndDocument)
                break
            else:
                yieldError("Unexpected token (expected scalar or line end): " &
                           $token.kind)
        of ylBlockLineEnd:
            case token.kind
            of yamlLineStart:
                state = ylBlockLineStart
            of yamlStreamEnd:
                closeAllLevels()
                yield YamlParserEvent(kind: yamlEndDocument)
                break
            else:
                yieldError("Unexpected token (expected line end):" &
                           $token.kind)
        else:
            discard
        
        token = nextToken(lex)