# NimYAML - YAML implementation in Nim # (c) Copyright 2016 Felix Krause # # See the file "copying.txt", included in this # distribution, for details about the copyright. ## ================== ## Module yaml.parser ## ================== ## ## This is the low-level parser API. A ``YamlParser`` enables you to parse any ## non-nil string or Stream object as YAML character stream. import tables, strutils, macros, streams import taglib, stream, ../private/lex, ../private/internal type WarningCallback* = proc(line, column: int, lineContent: string, message: string) ## Callback for parser warnings. Currently, this callback may be called ## on two occasions while parsing a YAML document stream: ## ## - If the version number in the ``%YAML`` directive does not match ## ``1.2``. ## - If there is an unknown directive encountered. YamlParser* = ref object ## A parser object. Retains its ``TagLibrary`` across calls to ## `parse <#parse,YamlParser,Stream>`_. Can be used ## to access anchor names while parsing a YAML character stream, but ## only until the document goes out of scope (i.e. until ## ``yamlEndDocument`` is yielded). tagLib: TagLibrary callback: WarningCallback anchors: Table[string, AnchorId] FastParseLevelKind = enum fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey, fplSinglePairValue, fplDocument FastParseLevel = object kind: FastParseLevelKind indentation: int ParserContext = ref object of YamlStream p: YamlParser lex: YamlLexer storedState: proc(s: YamlStream, e: var YamlStreamEvent): bool atSequenceItem: bool flowdepth: int ancestry: seq[FastParseLevel] level: FastParseLevel tag: TagId anchor: AnchorId shorthands: Table[string, string] nextAnchorId: AnchorId newlines: int explicitFlowKey: bool plainScalarStart: tuple[line, column: int] LevelEndResult = enum lerNothing, lerOne, lerAdditionalMapEnd YamlLoadingError* = object of Exception ## Base class for all exceptions that may be raised during the process ## of loading a YAML character stream. line*: int ## line number (1-based) where the error was encountered column*: int ## column number (1-based) where the error was encountered lineContent*: string ## \ ## content of the line where the error was encountered. Includes a ## second line with a marker ``^`` at the position where the error ## was encountered. YamlParserError* = object of YamlLoadingError ## A parser error is raised if the character stream that is parsed is ## not a valid YAML character stream. This stream cannot and will not be ## parsed wholly nor partially and all events that have been emitted by ## the YamlStream the parser provides should be discarded. ## ## A character stream is invalid YAML if and only if at least one of the ## following conditions apply: ## ## - There are invalid characters in an element whose contents is ## restricted to a limited set of characters. For example, there are ## characters in a tag URI which are not valid URI characters. ## - An element has invalid indentation. This can happen for example if ## a block list element indicated by ``"- "`` is less indented than ## the element in the previous line, but there is no block sequence ## list open at the same indentation level. ## - The YAML structure is invalid. For example, an explicit block map ## indicated by ``"? "`` and ``": "`` may not suddenly have a block ## sequence item (``"- "``) at the same indentation level. Another ## possible violation is closing a flow style object with the wrong ## closing character (``}``, ``]``) or not closing it at all. ## - A custom tag shorthand is used that has not previously been ## declared with a ``%TAG`` directive. ## - Multiple tags or anchors are defined for the same node. ## - An alias is used which does not map to any anchor that has ## previously been declared in the same document. ## - An alias has a tag or anchor associated with it. ## ## Some elements in this list are vague. For a detailed description of a ## valid YAML character stream, see the YAML specification. proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(), callback: WarningCallback = nil): YamlParser = ## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called ## whenever the parser yields a warning. new(result) result.tagLib = tagLib result.callback = callback template debug(message: string) {.dirty.} = when defined(yamlDebug): try: styledWriteLine(stdout, fgBlue, message) except IOError: discard proc generateError(c: ParserContext, message: string): ref YamlParserError {.raises: [].} = result = newException(YamlParserError, message) (result.line, result.column) = c.lex.curStartPos result.lineContent = c.lex.getTokenLine() proc illegalToken(c: ParserContext, expected: string = ""): ref YamlParserError {.raises: [].} = var msg = "Illegal token" if expected.len > 0: msg.add(" (expected " & expected & ")") msg.add(": " & $c.lex.cur) result = c.generateError(msg) proc callCallback(c: ParserContext, msg: string) {.raises: [YamlParserError].} = try: if not isNil(c.p.callback): c.p.callback(c.lex.curStartPos.line, c.lex.curStartPos.column, c.lex.getTokenLine(), msg) except: var e = newException(YamlParserError, "Warning callback raised exception: " & getCurrentExceptionMsg()) e.parent = getCurrentException() raise e proc initLevel(k: FastParseLevelKind): FastParseLevel {.raises: [], inline.} = FastParseLevel(kind: k, indentation: UnknownIndentation) proc emptyScalar(c: ParserContext): YamlStreamEvent {.raises: [], inline.} = result = scalarEvent("", c.tag, c.anchor) c.tag = yTagQuestionMark c.anchor = yAnchorNone proc currentScalar(c: ParserContext, e: var YamlStreamEvent) {.raises: [], inline.} = e = YamlStreamEvent(kind: yamlScalar, scalarTag: c.tag, scalarAnchor: c.anchor) shallowCopy(e.scalarContent, c.lex.buf) c.lex.buf = cast[string not nil](newStringOfCap(256)) c.tag = yTagQuestionMark c.anchor = yAnchorNone proc objectStart(c: ParserContext, k: static[YamlStreamEventKind], single: bool = false): YamlStreamEvent {.raises: [].} = yAssert(c.level.kind == fplUnknown) when k == yamlStartMap: result = startMapEvent(c.tag, c.anchor) if single: debug("started single-pair map at " & (if c.level.indentation == UnknownIndentation: $c.lex.indentation else: $c.level.indentation)) c.level.kind = fplSinglePairKey else: debug("started map at " & (if c.level.indentation == UnknownIndentation: $c.lex.indentation else: $c.level.indentation)) c.level.kind = fplMapKey else: result = startSeqEvent(c.tag, c.anchor) debug("started sequence at " & (if c.level.indentation == UnknownIndentation: $c.lex.indentation else: $c.level.indentation)) c.level.kind = fplSequence c.tag = yTagQuestionMark c.anchor = yAnchorNone if c.level.indentation == UnknownIndentation: c.level.indentation = c.lex.indentation c.ancestry.add(c.level) c.level = initLevel(fplUnknown) proc initDocValues(c: ParserContext) {.raises: [].} = c.shorthands = initTable[string, string]() c.p.anchors = initTable[string, AnchorId]() c.shorthands["!"] = "!" c.shorthands["!!"] = "tag:yaml.org,2002:" c.nextAnchorId = 0.AnchorId c.level = initLevel(fplUnknown) c.tag = yTagQuestionMark c.anchor = yAnchorNone c.ancestry.add(FastParseLevel(kind: fplDocument, indentation: -1)) proc advance(c: ParserContext) {.inline.} = try: c.lex.next() except YamlLexerError: let e = (ref YamlLexerError)(getCurrentException()) let pe = newException(YamlParserError, e.msg) pe.line = e.line pe.column = e.column pe.lineContent = e.lineContent raise pe proc handleAnchor(c: ParserContext) {.raises: [YamlParserError].} = if c.level.kind != fplUnknown: raise c.generateError("Unexpected token") if c.anchor != yAnchorNone: raise c.generateError("Only one anchor is allowed per node") c.anchor = c.nextAnchorId c.p.anchors[c.lex.buf] = c.anchor c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1) c.lex.buf.setLen(0) c.advance() proc handleTagHandle(c: ParserContext) {.raises: [YamlParserError].} = if c.level.kind != fplUnknown: raise c.generateError("Unexpected tag handle") if c.tag != yTagQuestionMark: raise c.generateError("Only one tag handle is allowed per node") if c.lex.cur == ltTagHandle: var tagUri = "" try: tagUri.add(c.shorthands[c.lex.buf[0..c.lex.shorthandEnd]]) tagUri.add(c.lex.buf[c.lex.shorthandEnd + 1 .. ^1]) except KeyError: raise c.generateError( "Undefined tag shorthand: " & c.lex.buf[0..c.lex.shorthandEnd]) try: c.tag = c.p.tagLib.tags[tagUri] except KeyError: c.tag = c.p.tagLib.registerUri(tagUri) else: try: c.tag = c.p.tagLib.tags[c.lex.buf] except KeyError: c.tag = c.p.tagLib.registerUri(c.lex.buf) c.lex.buf.setLen(0) c.advance() proc handlePossibleMapStart(c: ParserContext, e: var YamlStreamEvent, flow: bool = false, single: bool = false): bool = result = false if c.level.indentation == UnknownIndentation: if c.lex.isImplicitKeyStart(): e = c.objectStart(yamlStartMap, single) result = true c.level.indentation = c.lex.indentation proc handleMapKeyIndicator(c: ParserContext, e: var YamlStreamEvent): bool = result = false case c.level.kind of fplUnknown: e = c.objectStart(yamlStartMap) result = true of fplMapValue: if c.level.indentation != c.lex.indentation: raise c.generateError("Invalid p.indentation of map key indicator " & "(expected" & $c.level.indentation & ", got " & $c.lex.indentation & ")") e = scalarEvent("", yTagQuestionMark, yAnchorNone) result = true c.level.kind = fplMapKey c.ancestry.add(c.level) c.level = initLevel(fplUnknown) of fplMapKey: if c.level.indentation != c.lex.indentation: raise c.generateError("Invalid p.indentation of map key indicator") c.ancestry.add(c.level) c.level = initLevel(fplUnknown) of fplSequence: raise c.generateError("Unexpected map key indicator (expected '- ')") of fplSinglePairKey, fplSinglePairValue, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) c.advance() if c.lex.cur != ltIndentation: # this enables the parser to properly parse compact structures, like # ? - a # - b # and such. At the first `-`, the indentation must equal its level to be # parsed properly. c.lex.indentation = c.lex.curStartPos.column - 1 proc handleBlockSequenceIndicator(c: ParserContext, e: var YamlStreamEvent): bool = result = false case c.level.kind of fplUnknown: e = c.objectStart(yamlStartSeq) result = true of fplSequence: if c.level.indentation != c.lex.indentation: raise c.generateError( "Invalid p.indentation of block sequence indicator (expected " & $c.level.indentation & ", got " & $c.lex.indentation & ")") c.ancestry.add(c.level) c.level = initLevel(fplUnknown) else: raise c.generateError("Illegal sequence item in map") c.advance() if c.lex.cur != ltIndentation: # see comment in previous proc, this time with structures like # - - a # - b c.lex.indentation = c.lex.curStartPos.column - 1 proc handleBlockItemStart(c: ParserContext, e: var YamlStreamEvent): bool = result = false case c.level.kind of fplUnknown: result = c.handlePossibleMapStart(e) of fplSequence: raise c.generateError( "Unexpected token (expected block sequence indicator)") of fplMapKey: c.ancestry.add(c.level) c.level = FastParseLevel(kind: fplUnknown, indentation: c.lex.indentation) of fplMapValue: e = emptyScalar(c) result = true c.level.kind = fplMapKey c.ancestry.add(c.level) c.level = FastParseLevel(kind: fplUnknown, indentation: c.lex.indentation) of fplSinglePairKey, fplSinglePairValue, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) proc handleFlowItemStart(c: ParserContext, e: var YamlStreamEvent): bool = if c.level.kind == fplUnknown and c.ancestry[c.ancestry.high].kind == fplSequence: result = c.handlePossibleMapStart(e, true, true) else: result = false proc handleFlowPlainScalar(c: ParserContext) = while c.lex.cur in {ltScalarPart, ltEmptyLine}: c.lex.newlines.inc() c.advance() c.lex.newlines = 0 proc lastTokenContext(s: YamlStream, line, column: var int, lineContent: var string): bool = let c = ParserContext(s) line = c.lex.curStartPos.line column = c.lex.curStartPos.column lineContent = c.lex.getTokenLine(true) result = true # --- macros for defining parser states --- template capitalize(s: string): string = when declared(strutils.capitalizeAscii): strutils.capitalizeAscii(s) else: strutils.capitalize(s) macro parserStates(names: varargs[untyped]): typed = ## generates proc declaration for each state in list like this: ## ## proc name(s: YamlStream, e: var YamlStreamEvent): ## bool {.raises: [YamlParserError].} result = newStmtList() for name in names: let nameId = newIdentNode("state" & capitalize($name.ident)) result.add(newProc(nameId, [ident("bool"), newIdentDefs(ident("s"), ident("YamlStream")), newIdentDefs(ident("e"), newNimNode(nnkVarTy).add( ident("YamlStreamEvent")))], newEmptyNode())) result[0][4] = newNimNode(nnkPragma).add(newNimNode(nnkExprColonExpr).add( ident("raises"), newNimNode(nnkBracket).add(ident("YamlParserError"), ident("YamlLexerError")))) proc processStateAsgns(source, target: NimNode) {.compileTime.} = ## copies children of source to target and replaces all assignments ## `state = [name]` with the appropriate code for changing states. for child in source.children: if child.kind == nnkAsgn and child[0].kind == nnkIdent: if $child[0].ident == "state": assert child[1].kind == nnkIdent var newNameId: NimNode if child[1].kind == nnkIdent and $child[1].ident == "stored": newNameId = newDotExpr(ident("c"), ident("storedState")) else: newNameId = newIdentNode("state" & capitalize($child[1].ident)) target.add(newAssignment(newDotExpr( newIdentNode("s"), newIdentNode("nextImpl")), newNameId)) continue elif $child[0].ident == "stored": assert child[1].kind == nnkIdent let newNameId = newIdentNode("state" & capitalize($child[1].ident)) target.add(newAssignment(newDotExpr(newIdentNode("c"), newIdentNode("storedState")), newNameId)) continue var processed = copyNimNode(child) processStateAsgns(child, processed) target.add(processed) macro parserState(name: untyped, impl: untyped): typed = ## Creates a parser state. Every parser state is a proc with the signature ## ## proc(s: YamlStream, e: var YamlStreamEvent): ## bool {.raises: [YamlParserError].} ## ## The proc name will be prefixed with "state" and the original name will be ## capitalized, so a state "foo" will yield a proc named "stateFoo". ## ## Inside the proc, you have access to the ParserContext with the let variable ## `c`. You can change the parser state by a assignment `state = [newState]`. ## The [newState] must have been declared with states(...) previously. let nameStr = $name.ident nameId = newIdentNode("state" & capitalize(nameStr)) var procImpl = quote do: debug("state: " & `nameStr`) procImpl.add(newLetStmt(ident("c"), newCall("ParserContext", ident("s")))) procImpl.add(newAssignment(newIdentNode("result"), newLit(false))) assert impl.kind == nnkStmtList processStateAsgns(impl, procImpl) result = newProc(nameId, [ident("bool"), newIdentDefs(ident("s"), ident("YamlStream")), newIdentDefs(ident("e"), newNimNode(nnkVarTy).add(ident("YamlStreamEvent")))], procImpl) # --- parser states --- parserStates(initial, blockLineStart, blockObjectStart, blockAfterObject, scalarEnd, plainScalarEnd, objectEnd, expectDocEnd, startDoc, afterDocument, closeMoreIndentedLevels, afterPlainScalarYield, emitEmptyScalar, tagHandle, anchor, alias, flow, leaveFlowMap, leaveFlowSeq, flowAfterObject, leaveFlowSinglePairMap) proc closeEverything(c: ParserContext) = c.lex.indentation = -1 c.nextImpl = stateCloseMoreIndentedLevels proc endLevel(c: ParserContext, e: var YamlStreamEvent): LevelEndResult = result = lerOne case c.level.kind of fplSequence: e = endSeqEvent() of fplMapKey: e = endMapEvent() of fplMapValue, fplSinglePairValue: e = emptyScalar(c) c.level.kind = fplMapKey result = lerAdditionalMapEnd of fplUnknown: if c.ancestry.len > 1: e = emptyScalar(c) # don't yield scalar for empty doc else: result = lerNothing of fplDocument: e = endDocEvent() of fplSinglePairKey: internalError("Unexpected level kind: " & $c.level.kind) proc handleMapValueIndicator(c: ParserContext, e: var YamlStreamEvent): bool = result = false case c.level.kind of fplUnknown: if c.level.indentation == UnknownIndentation: e = c.objectStart(yamlStartMap) result = true c.storedState = c.nextImpl c.nextImpl = stateEmitEmptyScalar else: e = emptyScalar(c) result = true c.ancestry[c.ancestry.high].kind = fplMapValue of fplMapKey: if c.level.indentation != c.lex.indentation: raise c.generateError("Invalid p.indentation of map key indicator") e = scalarEvent("", yTagQuestionMark, yAnchorNone) result = true c.level.kind = fplMapValue c.ancestry.add(c.level) c.level = initLevel(fplUnknown) of fplMapValue: if c.level.indentation != c.lex.indentation: raise c.generateError("Invalid p.indentation of map key indicator") c.ancestry.add(c.level) c.level = initLevel(fplUnknown) of fplSequence: raise c.generateError("Unexpected map value indicator (expected '- ')") of fplSinglePairKey, fplSinglePairValue, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) c.advance() if c.lex.cur != ltIndentation: # see comment in handleMapKeyIndicator, this time with structures like # a: - a # - b c.lex.indentation = c.lex.curStartPos.column - 1 template handleObjectEnd(c: ParserContext, mayHaveEmptyValue: bool = false): bool = var result = false c.level = c.ancestry.pop() when mayHaveEmptyValue: if c.level.kind == fplSinglePairValue: result = true c.level = c.ancestry.pop() case c.level.kind of fplMapKey: c.level.kind = fplMapValue of fplSinglePairKey: c.level.kind = fplSinglePairValue of fplMapValue: c.level.kind = fplMapKey of fplSequence, fplDocument: discard of fplUnknown, fplSinglePairValue: internalError("Unexpected level kind: " & $c.level.kind) result proc leaveFlowLevel(c: ParserContext, e: var YamlStreamEvent): bool = c.flowdepth.dec() result = (c.endLevel(e) == lerOne) # lerAdditionalMapEnd cannot happen if c.flowdepth == 0: c.lex.setFlow(false) c.storedState = stateBlockAfterObject else: c.storedState = stateFlowAfterObject c.nextImpl = stateObjectEnd c.advance() parserState initial: case c.lex.cur of ltYamlDirective: c.advance() assert c.lex.cur == ltYamlVersion, $c.lex.cur if c.lex.buf != "1.2": c.callCallback("Version is not 1.2, but " & c.lex.buf) c.lex.buf.setLen(0) c.advance() of ltTagDirective: c.advance() assert c.lex.cur == ltTagShorthand var tagShorthand: string shallowCopy(tagShorthand, c.lex.buf) c.lex.buf = "" c.advance() assert c.lex.cur == ltTagUri c.shorthands[tagShorthand] = c.lex.buf c.lex.buf.setLen(0) c.advance() of ltUnknownDirective: c.callCallback("Unknown directive: " & c.lex.buf) c.lex.buf.setLen(0) c.advance() if c.lex.cur == ltUnknownDirectiveParams: c.lex.buf.setLen(0) c.advance() of ltIndentation: e = startDocEvent() result = true state = blockObjectStart of ltStreamEnd: c.isFinished = true of ltDirectivesEnd: e = startDocEvent() result = true c.advance() state = blockObjectStart of ltDocumentEnd: c.advance() state = afterDocument else: internalError("Unexpected lexer token: " & $c.lex.cur) parserState blockLineStart: case c.lex.cur of ltIndentation: c.advance() of ltEmptyLine: c.advance() of ltStreamEnd: c.closeEverything() stored = afterDocument else: if c.lex.indentation <= c.ancestry[^1].indentation: state = closeMoreIndentedLevels stored = blockObjectStart else: state = blockObjectStart parserState blockObjectStart: case c.lex.cur of ltEmptyLine: c.advance() of ltIndentation: c.advance() c.level.indentation = UnknownIndentation state = blockLineStart of ltDirectivesEnd: c.closeEverything() stored = startDoc of ltDocumentEnd: c.advance() c.closeEverything() stored = afterDocument of ltMapKeyInd: result = c.handleMapKeyIndicator(e) of ltMapValInd: result = c.handleMapValueIndicator(e) of ltQuotedScalar: result = c.handleBlockItemStart(e) c.advance() state = scalarEnd of ltBlockScalarHeader: c.lex.indentation = c.ancestry[^1].indentation c.advance() assert c.lex.cur == ltBlockScalar if c.level.indentation == UnknownIndentation: c.level.indentation = c.lex.indentation c.advance() state = scalarEnd of ltScalarPart: result = c.handleBlockItemStart(e) c.plainScalarStart = c.lex.curStartPos while true: c.advance() case c.lex.cur of ltIndentation: if c.lex.indentation <= c.ancestry[^1].indentation: break c.lex.newlines.inc() of ltScalarPart: discard of ltEmptyLine: c.lex.newlines.inc() else: break c.lex.newlines = 0 state = plainScalarEnd stored = blockAfterObject of ltSeqItemInd: result = c.handleBlockSequenceIndicator(e) of ltTagHandle, ltLiteralTag: result = c.handleBlockItemStart(e) state = tagHandle stored = blockObjectStart of ltAnchor: result = c.handleBlockItemStart(e) state = anchor stored = blockObjectStart of ltAlias: result = c.handleBlockItemStart(e) state = alias stored = blockAfterObject of ltBraceOpen, ltBracketOpen: result = c.handleBlockItemStart(e) c.lex.setFlow(true) state = flow of ltStreamEnd: c.closeEverything() stored = afterDocument else: raise c.generateError("Unexpected token: " & $c.lex.cur) parserState scalarEnd: if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark c.currentScalar(e) result = true state = objectEnd stored = blockAfterObject parserState plainScalarEnd: c.currentScalar(e) result = true c.lastTokenContextImpl = proc(s: YamlStream, line, column: var int, lineContent: var string): bool {.raises: [].} = let c = ParserContext(s) (line, column) = c.plainScalarStart lineContent = c.lex.getTokenLine(c.plainScalarStart, true) result = true state = afterPlainScalarYield stored = blockAfterObject parserState afterPlainScalarYield: c.lastTokenContextImpl = lastTokenContext state = objectEnd parserState blockAfterObject: case c.lex.cur of ltIndentation, ltEmptyLine: c.advance() state = blockLineStart of ltMapValInd: case c.level.kind of fplUnknown: e = c.objectStart(yamlStartMap) result = true of fplMapKey: e = scalarEvent("", yTagQuestionMark, yAnchorNone) result = true c.level.kind = fplMapValue c.ancestry.add(c.level) c.level = initLevel(fplUnknown) of fplMapValue: c.level.kind = fplMapValue c.ancestry.add(c.level) c.level = initLevel(fplUnknown) of fplSequence: raise c.illegalToken("sequence item") of fplSinglePairKey, fplSinglePairValue, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) c.advance() state = blockObjectStart of ltDirectivesEnd: c.closeEverything() stored = startDoc of ltStreamEnd: c.closeEverything() stored = afterDocument else: raise c.illegalToken("':', comment or line end") parserState objectEnd: if c.handleObjectEnd(true): e = endMapEvent() result = true if c.level.kind == fplDocument: state = expectDocEnd else: state = stored parserState expectDocEnd: case c.lex.cur of ltIndentation, ltEmptyLine: c.advance() of ltDirectivesEnd: e = endDocEvent() result = true state = startDoc c.ancestry.setLen(0) of ltDocumentEnd: e = endDocEvent() result = true state = afterDocument c.advance() of ltStreamEnd: e = endDocEvent() result = true c.isFinished = true else: raise c.generateError("Unexpected token (expected document end): " & $c.lex.cur) parserState startDoc: c.initDocValues() e = startDocEvent() result = true c.advance() state = blockObjectStart parserState afterDocument: case c.lex.cur of ltStreamEnd: c.isFinished = true of ltEmptyLine: c.advance() else: c.initDocValues() state = initial parserState closeMoreIndentedLevels: if c.ancestry.len > 0: let parent = c.ancestry[c.ancestry.high] if parent.indentation >= c.lex.indentation: if c.lex.cur == ltSeqItemInd: if (c.lex.indentation == c.level.indentation and c.level.kind == fplSequence) or (c.lex.indentation == parent.indentation and c.level.kind == fplUnknown and parent.kind != fplSequence): state = stored debug("Not closing because sequence indicator") return false debug("Closing because parent.indentation (" & $parent.indentation & ") >= indentation(" & $c.lex.indentation & ")") case c.endLevel(e) of lerNothing: discard of lerOne: result = true of lerAdditionalMapEnd: return true discard c.handleObjectEnd(false) return result debug("Not closing level because parent.indentation (" & $parent.indentation & ") < indentation(" & $c.lex.indentation & ")") if c.level.kind == fplDocument: state = expectDocEnd else: state = stored elif c.lex.indentation == c.level.indentation: debug("Closing document") let res = c.endLevel(e) yAssert(res == lerOne) result = true state = stored else: state = stored parserState emitEmptyScalar: e = scalarEvent("", yTagQuestionMark, yAnchorNone) result = true state = stored parserState tagHandle: c.handleTagHandle() state = stored parserState anchor: c.handleAnchor() state = stored parserState alias: if c.level.kind != fplUnknown: raise c.generateError("Unexpected token") if c.anchor != yAnchorNone or c.tag != yTagQuestionMark: raise c.generateError("Alias may not have anchor or tag") var id: AnchorId try: id = c.p.anchors[c.lex.buf] except KeyError: raise c.generateError("Unknown anchor") c.lex.buf.setLen(0) e = aliasEvent(id) c.advance() result = true state = objectEnd parserState flow: case c.lex.cur of ltBraceOpen: if c.handleFlowItemStart(e): return true e = c.objectStart(yamlStartMap) result = true c.flowdepth.inc() c.explicitFlowKey = false c.advance() of ltBracketOpen: if c.handleFlowItemStart(e): return true e = c.objectStart(yamlStartSeq) result = true c.flowdepth.inc() c.advance() of ltBraceClose: yAssert(c.level.kind == fplUnknown) c.level = c.ancestry.pop() state = leaveFlowMap of ltBracketClose: yAssert(c.level.kind == fplUnknown) c.level = c.ancestry.pop() state = leaveFlowSeq of ltComma: yAssert(c.level.kind == fplUnknown) c.level = c.ancestry.pop() case c.level.kind of fplSequence: e = c.emptyScalar() result = true of fplMapValue: e = c.emptyScalar() result = true c.level.kind = fplMapKey c.explicitFlowKey = false of fplMapKey: e = c.emptyScalar() c.level.kind = fplMapValue return true of fplSinglePairValue: e = c.emptyScalar() result = true c.level = c.ancestry.pop() state = leaveFlowSinglePairMap stored = flow of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) c.ancestry.add(c.level) c.level = initLevel(fplUnknown) c.advance() of ltMapValInd: c.level = c.ancestry.pop() case c.level.kind of fplSequence: e = startMapEvent(c.tag, c.anchor) result = true debug("started single-pair map at " & (if c.level.indentation == UnknownIndentation: $c.lex.indentation else: $c.level.indentation)) c.tag = yTagQuestionMark c.anchor = yAnchorNone if c.level.indentation == UnknownIndentation: c.level.indentation = c.lex.indentation c.ancestry.add(c.level) c.level = initLevel(fplSinglePairKey) of fplMapValue, fplSinglePairValue: raise c.generateError("Unexpected token (expected ',')") of fplMapKey: e = c.emptyScalar() result = true c.level.kind = fplMapValue of fplSinglePairKey: e = c.emptyScalar() result = true c.level.kind = fplSinglePairValue of fplUnknown, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) if c.level.kind != fplSinglePairKey: c.advance() c.ancestry.add(c.level) c.level = initLevel(fplUnknown) of ltQuotedScalar: if c.handleFlowItemStart(e): return true if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark c.currentScalar(e) result = true state = objectEnd stored = flowAfterObject c.advance() of ltTagHandle, ltLiteralTag: if c.handleFlowItemStart(e): return true c.handleTagHandle() of ltAnchor: if c.handleFlowItemStart(e): return true c.handleAnchor() of ltAlias: state = alias stored = flowAfterObject of ltMapKeyInd: if c.explicitFlowKey: raise c.generateError("Duplicate '?' in flow mapping") elif c.level.kind == fplUnknown: case c.ancestry[c.ancestry.high].kind of fplMapKey, fplMapValue, fplDocument: discard of fplSequence: e = c.objectStart(yamlStartMap, true) result = true else: raise c.generateError("Unexpected token") c.explicitFlowKey = true c.advance() of ltScalarPart: if c.handleFlowItemStart(e): return true c.handleFlowPlainScalar() c.currentScalar(e) result = true state = objectEnd stored = flowAfterObject else: raise c.generateError("Unexpected toked: " & $c.lex.cur) parserState leaveFlowMap: case c.level.kind of fplMapValue: e = c.emptyScalar() c.level.kind = fplMapKey return true of fplMapKey: if c.tag != yTagQuestionMark or c.anchor != yAnchorNone or c.explicitFlowKey: e = c.emptyScalar() c.level.kind = fplMapValue c.explicitFlowKey = false return true of fplSequence: raise c.generateError("Unexpected token (expected ']')") of fplSinglePairValue: raise c.generateError("Unexpected token (expected ']')") of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) result = c.leaveFlowLevel(e) parserState leaveFlowSeq: case c.level.kind of fplSequence: if c.tag != yTagQuestionMark or c.anchor != yAnchorNone: e = c.emptyScalar() return true of fplSinglePairValue: e = c.emptyScalar() c.level = c.ancestry.pop() state = leaveFlowSinglePairMap stored = leaveFlowSeq return true of fplMapKey, fplMapValue: raise c.generateError("Unexpected token (expected '}')") of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) result = c.leaveFlowLevel(e) parserState leaveFlowSinglePairMap: e = endMapEvent() result = true state = stored parserState flowAfterObject: case c.lex.cur of ltBracketClose: case c.level.kind of fplSequence: discard of fplMapKey, fplMapValue: raise c.generateError("Unexpected token (expected '}')") of fplSinglePairValue: c.level = c.ancestry.pop() yAssert(c.level.kind == fplSequence) e = endMapEvent() return true of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) result = c.leaveFlowLevel(e) of ltBraceClose: case c.level.kind of fplMapKey, fplMapValue: discard of fplSequence, fplSinglePairValue: raise c.generateError("Unexpected token (expected ']')") of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) result = c.leaveFlowLevel(e) of ltComma: case c.level.kind of fplSequence: discard of fplMapValue: e = scalarEvent("", yTagQuestionMark, yAnchorNone) result = true c.level.kind = fplMapKey c.explicitFlowKey = false of fplSinglePairValue: c.level = c.ancestry.pop() yAssert(c.level.kind == fplSequence) e = endMapEvent() result = true of fplMapKey: c.explicitFlowKey = false of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) c.ancestry.add(c.level) c.level = initLevel(fplUnknown) state = flow c.advance() of ltMapValInd: case c.level.kind of fplSequence, fplMapKey: raise c.generateError("Unexpected token (expected ',')") of fplMapValue, fplSinglePairValue: discard of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) c.ancestry.add(c.level) c.level = initLevel(fplUnknown) state = flow c.advance() of ltStreamEnd: raise c.generateError("Unclosed flow content") else: raise c.generateError("Unexpected content (expected flow indicator)") # --- parser initialization --- proc init(c: ParserContext, p: YamlParser) = c.basicInit(lastTokenContext) c.p = p c.ancestry = newSeq[FastParseLevel]() c.initDocValues() c.flowdepth = 0 c.nextImpl = stateInitial c.explicitFlowKey = false c.advance() proc parse*(p: YamlParser, s: Stream): YamlStream {.raises: [YamlParserError].} = ## Parse the given stream as YAML character stream. let c = new(ParserContext) try: c.lex = newYamlLexer(s) except: let e = newException(YamlParserError, "Error while opening stream: " & getCurrentExceptionMsg()) e.parent = getCurrentException() e.line = 1 e.column = 1 e.lineContent = "" raise e c.init(p) result = c proc parse*(p: YamlParser, str: string): YamlStream {.raises: [YamlParserError].} = ## Parse the given string as YAML character stream. let c = new(ParserContext) c.lex = newYamlLexer(str) c.init(p) result = c