From 345f86c52eef775482dd6f7c76275bab1a7f8d50 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Wed, 14 Sep 2016 13:37:56 +0200 Subject: [PATCH] Pass almost all test in yaml test suite --- private/lex.nim | 8 ++++++-- private/parse.nim | 40 +++++++++++++++++++++------------------- private/stream.nim | 1 + 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/private/lex.nim b/private/lex.nim index d0ea56e..10d8e94 100644 --- a/private/lex.nim +++ b/private/lex.nim @@ -911,15 +911,18 @@ proc blockScalar[T](lex: YamlLexer): bool = lex.lineStartState = insideDoc[T] lex.cur = ltBlockScalar result = true + echo "exiting block scalar with indentation=" & $lex.indentation proc indentationAfterBlockScalar[T](lex: YamlLexer): bool = - if lex.c == '#': + if lex.indentation == 0: + lex.nextState = lex.insideDocImpl + elif lex.c == '#': lex.nextState = expectLineEnd[T] result = false else: lex.cur = ltIndentation result = true - lex.nextState = lex.lineStartState + lex.nextState = lex.insideLineImpl proc dirEndAfterBlockScalar[T](lex: YamlLexer): bool = lex.cur = ltDirectivesEnd @@ -1139,6 +1142,7 @@ proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer = proc next*(lex: YamlLexer) = while not lex.nextState(lex): discard + debug("lexer -> " & $lex.cur) proc setFlow*(lex: YamlLexer, value: bool) = lex.inFlow = value diff --git a/private/parse.nim b/private/parse.nim index 4992fab..a9c4862 100644 --- a/private/parse.nim +++ b/private/parse.nim @@ -202,8 +202,8 @@ proc handleMapKeyIndicator(c: ParserContext, e: var YamlStreamEvent): bool = c.advance() if c.lex.cur != ltIndentation: # this enables the parser to properly parse compact structures, like - # a: - a - # - b + # ? - a + # - b # and such. At the first `-`, the indentation must equal its level to be # parsed properly. c.lex.indentation = c.lex.curStartPos.column - 1 @@ -339,7 +339,7 @@ macro parserState(name: untyped, impl: untyped): typed = parserStates(initial, blockLineStart, blockObjectStart, blockAfterObject, scalarEnd, plainScalarEnd, objectEnd, expectDocEnd, startDoc, - endDoc, afterDocument, closeStream, closeMoreIndentedLevels, + afterDocument, closeStream, closeMoreIndentedLevels, emitEmptyScalar, tagHandle, anchor, alias, flow, leaveFlowMap, leaveFlowSeq, flowAfterObject, leaveFlowSinglePairMap) @@ -400,6 +400,11 @@ proc handleMapValueIndicator(c: ParserContext, e: var YamlStreamEvent): bool = of fplSinglePairKey, fplSinglePairValue, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) c.advance() + if c.lex.cur != ltIndentation: + # see comment in handleMapKeyIndicator, this time with structures like + # a: - a + # - b + c.lex.indentation = c.lex.curStartPos.column - 1 template handleObjectEnd(c: ParserContext, mayHaveEmptyValue: bool = false): bool = @@ -427,6 +432,7 @@ proc leaveFlowLevel(c: ParserContext, e: var YamlStreamEvent): bool = else: c.storedState = stateFlowAfterObject c.nextImpl = stateObjectEnd + c.advance() parserState initial: case c.lex.cur @@ -466,9 +472,8 @@ parserState initial: c.advance() state = blockObjectStart of ltDocumentEnd: - e = startDocEvent() - result = true - state = endDoc + c.advance() + state = afterDocument else: internalError("Unexpected lexer token: " & $c.lex.cur) parserState blockLineStart: @@ -496,6 +501,7 @@ parserState blockObjectStart: c.closeEverything() stored = startDoc of ltDocumentEnd: + c.advance() c.closeEverything() stored = afterDocument of ltMapKeyInd: @@ -530,7 +536,7 @@ parserState blockObjectStart: stored = blockAfterObject of ltSeqItemInd: result = c.handleBlockSequenceIndicator(e) - of ltTagHandle: + of ltTagHandle, ltLiteralTag: result = c.handleBlockItemStart(e) state = tagHandle stored = blockObjectStart @@ -593,7 +599,6 @@ parserState blockAfterObject: of ltDirectivesEnd: c.closeEverything() stored = startDoc - c.advance() of ltStreamEnd: c.closeEverything() stored = afterDocument @@ -618,6 +623,7 @@ parserState expectDocEnd: e = endDocEvent() result = true state = afterDocument + c.advance() of ltStreamEnd: e = endDocEvent() result = true @@ -630,13 +636,9 @@ parserState startDoc: c.initDocValues() e = startDocEvent() result = true + c.advance() state = blockObjectStart -parserState endDoc: - e = endDocEvent() - result = true - state = initial - parserState afterDocument: case c.lex.cur of ltStreamEnd: c.isFinished = true @@ -647,7 +649,6 @@ parserState afterDocument: parserState closeStream: case c.level.kind - of fplUnknown: discard c.ancestry.pop() of fplDocument: discard else: case c.endLevel(e) @@ -670,6 +671,7 @@ parserState closeMoreIndentedLevels: (c.lex.indentation == parent.indentation and c.level.kind == fplUnknown and parent.kind != fplSequence): state = stored + debug("Not closing because sequence indicator") return false debug("Closing because parent.indentation (" & $parent.indentation & ") >= indentation(" & $c.lex.indentation & ")") @@ -679,9 +681,13 @@ parserState closeMoreIndentedLevels: of lerAdditionalMapEnd: return true discard c.handleObjectEnd(false) return result + debug("Not closing level because parent.indentation (" & + $parent.indentation & ") < indentation(" & $c.lex.indentation & + ")") if c.level.kind == fplDocument: state = expectDocEnd else: state = stored elif c.lex.indentation == c.level.indentation: + debug("Closing document") let res = c.endLevel(e) yAssert(res == lerOne) result = true @@ -733,12 +739,10 @@ parserState flow: of ltBraceClose: yAssert(c.level.kind == fplUnknown) c.level = c.ancestry.pop() - c.advance() state = leaveFlowMap of ltBracketClose: yAssert(c.level.kind == fplUnknown) c.level = c.ancestry.pop() - c.advance() state = leaveFlowSeq of ltComma: yAssert(c.level.kind == fplUnknown) @@ -805,7 +809,7 @@ parserState flow: state = objectEnd stored = flowAfterObject c.advance() - of ltTagHandle: + of ltTagHandle, ltLiteralTag: if c.handleFlowItemStart(e): return true c.handleTagHandle() of ltAnchor: @@ -896,7 +900,6 @@ parserState flowAfterObject: of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) result = c.leaveFlowLevel(e) - c.advance() of ltBraceClose: case c.level.kind of fplMapKey, fplMapValue: discard @@ -905,7 +908,6 @@ parserState flowAfterObject: of fplUnknown, fplSinglePairKey, fplDocument: internalError("Unexpected level kind: " & $c.level.kind) result = c.leaveFlowLevel(e) - c.advance() of ltComma: case c.level.kind of fplSequence: discard diff --git a/private/stream.nim b/private/stream.nim index db98182..959728c 100644 --- a/private/stream.nim +++ b/private/stream.nim @@ -49,6 +49,7 @@ proc next*(s: YamlStream): YamlStreamEvent = if s.peeked: s.peeked = false shallowCopy(result, s.cached) + return else: yAssert(not s.isFinished) try: