status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Added fplDocument, fixes #11 

* Do not use -1 for unknown indentation, because the document level is -1.
 * Use const UnknownIndentation in code instead of magic number
 * Added fplDocument level which is used as root of the ancestry
 * Added debugFail() template to better locate internal errors
This commit is contained in:
Felix Krause 2016-03-19 22:43:03 +01:00
parent e45eb7b514
commit 4d4e3c94e1
2 changed files with 108 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

198
private/fastparse.nim
View File

@ -12,7 +12,7 @@ type
FastParseLevelKind = enum FastParseLevelKind = enum
fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey, fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey,
fplSinglePairValue, fplScalar fplSinglePairValue, fplScalar, fplDocument
FastParseLevel = object FastParseLevel = object
kind: FastParseLevelKind kind: FastParseLevelKind
@ -38,6 +38,7 @@ const
UTF8NonBreakingSpace = toUTF8(0xA0.Rune) UTF8NonBreakingSpace = toUTF8(0xA0.Rune)
UTF8LineSeparator = toUTF8(0x2028.Rune) UTF8LineSeparator = toUTF8(0x2028.Rune)
UTF8ParagraphSeparator = toUTF8(0x2029.Rune) UTF8ParagraphSeparator = toUTF8(0x2029.Rune)
UnknownIndentation = int.low
proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(), proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(),
callback: WarningCallback = nil): YamlParser = callback: WarningCallback = nil): YamlParser =
@ -59,6 +60,11 @@ template debug(message: string) {.dirty.} =
try: styledWriteLine(stdout, fgBlue, message) try: styledWriteLine(stdout, fgBlue, message)
except IOError: discard except IOError: discard
template debugFail() {.dirty.} =
when not defined(release):
echo "internal error at line: ", instantiationInfo().line
assert(false)
template parserError(message: string) {.dirty.} = template parserError(message: string) {.dirty.} =
var e = newException(YamlParserError, message) var e = newException(YamlParserError, message)
e.line = p.lexer.lineNumber e.line = p.lexer.lineNumber
@ -74,6 +80,9 @@ template lexerError(lx: BaseLexer, message: string) {.dirty.} =
repeat(' ', lx.getColNumber(lx.bufpos)) & "^\n" repeat(' ', lx.getColNumber(lx.bufpos)) & "^\n"
raise e raise e
template initLevel(k: FastParseLevelKind): FastParseLevel =
FastParseLevel(kind: k, indentation: UnknownIndentation)
template yieldEmptyScalar() {.dirty.} = template yieldEmptyScalar() {.dirty.} =
yield scalarEvent("", tag, anchor) yield scalarEvent("", tag, anchor)
tag = yTagQuestionMark tag = yTagQuestionMark
@ -81,10 +90,8 @@ template yieldEmptyScalar() {.dirty.} =
template yieldLevelEnd() {.dirty.} = template yieldLevelEnd() {.dirty.} =
case level.kind case level.kind
of fplSequence: of fplSequence: yield endSeqEvent()
yield endSeqEvent() of fplMapKey: yield endMapEvent()
of fplMapKey:
yield endMapEvent()
of fplMapValue, fplSinglePairValue: of fplMapValue, fplSinglePairValue:
yieldEmptyScalar() yieldEmptyScalar()
yield endMapEvent() yield endMapEvent()
@ -92,8 +99,9 @@ template yieldLevelEnd() {.dirty.} =
yield scalarEvent(content, tag, anchor) yield scalarEvent(content, tag, anchor)
tag = yTagQuestionMark tag = yTagQuestionMark
anchor = yAnchorNone anchor = yAnchorNone
of fplUnknown: yieldEmptyScalar() of fplUnknown:
of fplSinglePairKey: assert(false) if ancestry.len > 1: yieldEmptyScalar() # don't yield scalar for empty doc
of fplSinglePairKey, fplDocument: debugFail()
template handleLineEnd(insideDocument: bool) {.dirty.} = template handleLineEnd(insideDocument: bool) {.dirty.} =
case p.lexer.buf[p.lexer.bufpos] case p.lexer.buf[p.lexer.bufpos]
@ -106,23 +114,19 @@ template handleLineEnd(insideDocument: bool) {.dirty.} =
newlines.inc() newlines.inc()
template handleObjectEnd(nextState: FastParseState) {.dirty.} = template handleObjectEnd(nextState: FastParseState) {.dirty.} =
if ancestry.len == 0: level = ancestry.pop()
level.kind = fplUnknown if level.kind == fplSinglePairValue:
state = fpExpectDocEnd yield endMapEvent()
else:
level = ancestry.pop() level = ancestry.pop()
if level.kind == fplSinglePairValue: state = if level.kind == fplDocument: fpExpectDocEnd else: nextState
yield endMapEvent() tag = yTagQuestionMark
level = ancestry.pop() anchor = yAnchorNone
state = nextState case level.kind
tag = yTagQuestionMark of fplMapKey: level.kind = fplMapValue
anchor = yAnchorNone of fplSinglePairKey: level.kind = fplSinglePairValue
case level.kind of fplMapValue: level.kind = fplMapKey
of fplMapKey: level.kind = fplMapValue of fplSequence, fplDocument: discard
of fplSinglePairKey: level.kind = fplSinglePairValue of fplUnknown, fplScalar, fplSinglePairValue: debugFail()
of fplMapValue: level.kind = fplMapKey
of fplSequence: discard
of fplUnknown, fplScalar, fplSinglePairValue: assert(false)
template handleObjectStart(k: YamlStreamEventKind, single: bool = false) template handleObjectStart(k: YamlStreamEventKind, single: bool = false)
{.dirty.} = {.dirty.} =
@ -131,26 +135,28 @@ template handleObjectStart(k: YamlStreamEventKind, single: bool = false)
yield startMapEvent(tag, anchor) yield startMapEvent(tag, anchor)
if single: if single:
debug("started single-pair map at " & debug("started single-pair map at " &
(if level.indentation == -1: $indentation else: $level.indentation)) (if level.indentation == UnknownIndentation: $indentation else:
$level.indentation))
level.kind = fplSinglePairKey level.kind = fplSinglePairKey
else: else:
debug("started map at " & debug("started map at " &
(if level.indentation == -1: $indentation else: $level.indentation)) (if level.indentation == UnknownIndentation: $indentation else:
$level.indentation))
level.kind = fplMapKey level.kind = fplMapKey
else: else:
yield startSeqEvent(tag, anchor) yield startSeqEvent(tag, anchor)
debug("started sequence at " & (if level.indentation == -1: $indentation else: debug("started sequence at " &
$level.indentation)) (if level.indentation == UnknownIndentation: $indentation else:
$level.indentation))
level.kind = fplSequence level.kind = fplSequence
tag = yTagQuestionMark tag = yTagQuestionMark
anchor = yAnchorNone anchor = yAnchorNone
if level.indentation == -1: if level.indentation == UnknownIndentation: level.indentation = indentation
level.indentation = indentation
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
template closeMoreIndentedLevels(atSequenceItem: bool = false) {.dirty.} = template closeMoreIndentedLevels(atSequenceItem: bool = false) {.dirty.} =
while ancestry.len > 0: while level.kind != fplDocument:
let parent = ancestry[ancestry.high] let parent = ancestry[ancestry.high]
if parent.indentation >= indentation: if parent.indentation >= indentation:
when atSequenceItem: when atSequenceItem:
@ -163,11 +169,17 @@ template closeMoreIndentedLevels(atSequenceItem: bool = false) {.dirty.} =
yieldLevelEnd() yieldLevelEnd()
handleObjectEnd(state) handleObjectEnd(state)
else: break else: break
if level.kind == fplDocument: state = fpExpectDocEnd
template closeEverything() {.dirty.} = template closeEverything() {.dirty.} =
indentation = 0 indentation = 0
closeMoreIndentedLevels() closeMoreIndentedLevels()
if level.kind != fplUnknown: yieldLevelEnd() case level.kind
of fplUnknown: discard ancestry.pop()
of fplDocument: discard
else:
yieldLevelEnd()
discard ancestry.pop()
yield endDocEvent() yield endDocEvent()
template handleBlockSequenceIndicator() {.dirty.} = template handleBlockSequenceIndicator() {.dirty.} =
@ -178,7 +190,7 @@ template handleBlockSequenceIndicator() {.dirty.} =
if level.indentation != indentation: if level.indentation != indentation:
parserError("Invalid indentation of block sequence indicator") parserError("Invalid indentation of block sequence indicator")
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
else: parserError("Illegal sequence item in map") else: parserError("Illegal sequence item in map")
p.lexer.skipWhitespace() p.lexer.skipWhitespace()
indentation = p.lexer.getColNumber(p.lexer.bufpos) indentation = p.lexer.getColNumber(p.lexer.bufpos)
@ -193,17 +205,17 @@ template handleMapKeyIndicator() {.dirty.} =
yield scalarEvent("", yTagQuestionMark, yAnchorNone) yield scalarEvent("", yTagQuestionMark, yAnchorNone)
level.kind = fplMapKey level.kind = fplMapKey
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
of fplMapKey: of fplMapKey:
if level.indentation != indentation: if level.indentation != indentation:
parserError("Invalid indentation of map key indicator") parserError("Invalid indentation of map key indicator")
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
of fplSequence: of fplSequence:
parserError("Unexpected map key indicator (expected '- ')") parserError("Unexpected map key indicator (expected '- ')")
of fplScalar: of fplScalar:
parserError("Unexpected map key indicator (expected multiline scalar end)") parserError("Unexpected map key indicator (expected multiline scalar end)")
of fplSinglePairKey, fplSinglePairValue: assert(false) of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
p.lexer.skipWhitespace() p.lexer.skipWhitespace()
indentation = p.lexer.getColNumber(p.lexer.bufpos) indentation = p.lexer.getColNumber(p.lexer.bufpos)
@ -211,7 +223,7 @@ template handleMapValueIndicator() {.dirty.} =
startToken() startToken()
case level.kind case level.kind
of fplUnknown: of fplUnknown:
if level.indentation == -1: if level.indentation == UnknownIndentation:
handleObjectStart(yamlStartMap) handleObjectStart(yamlStartMap)
yield scalarEvent("", yTagQuestionMark, yAnchorNone) yield scalarEvent("", yTagQuestionMark, yAnchorNone)
else: yieldEmptyScalar() else: yieldEmptyScalar()
@ -222,18 +234,18 @@ template handleMapValueIndicator() {.dirty.} =
yield scalarEvent("", yTagQuestionMark, yAnchorNone) yield scalarEvent("", yTagQuestionMark, yAnchorNone)
level.kind = fplMapValue level.kind = fplMapValue
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
of fplMapValue: of fplMapValue:
if level.indentation != indentation: if level.indentation != indentation:
parserError("Invalid indentation of map key indicator") parserError("Invalid indentation of map key indicator")
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
of fplSequence: of fplSequence:
parserError("Unexpected map value indicator (expected '- ')") parserError("Unexpected map value indicator (expected '- ')")
of fplScalar: of fplScalar:
parserError( parserError(
"Unexpected map value indicator (expected multiline scalar end)") "Unexpected map value indicator (expected multiline scalar end)")
of fplSinglePairKey, fplSinglePairValue: assert(false) of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
p.lexer.skipWhitespace() p.lexer.skipWhitespace()
indentation = p.lexer.getColNumber(p.lexer.bufpos) indentation = p.lexer.getColNumber(p.lexer.bufpos)
@ -243,9 +255,10 @@ template initDocValues() {.dirty.} =
shorthands["!"] = "!" shorthands["!"] = "!"
shorthands["!!"] = "tag:yaml.org,2002:" shorthands["!!"] = "tag:yaml.org,2002:"
nextAnchorId = 0.AnchorId nextAnchorId = 0.AnchorId
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
tag = yTagQuestionMark tag = yTagQuestionMark
anchor = yAnchorNone anchor = yAnchorNone
ancestry.add(FastParseLevel(kind: fplDocument, indentation: -1))
template handleTagHandle() {.dirty.} = template handleTagHandle() {.dirty.} =
startToken() startToken()
@ -302,7 +315,7 @@ template leaveFlowLevel() {.dirty.} =
template handlePossibleMapStart(flow: bool = false, template handlePossibleMapStart(flow: bool = false,
single: bool = false) {.dirty.} = single: bool = false) {.dirty.} =
if level.indentation == -1: if level.indentation == UnknownIndentation:
var flowDepth = 0 var flowDepth = 0
var pos = p.lexer.bufpos var pos = p.lexer.bufpos
var recentJsonStyle = false var recentJsonStyle = false
@ -341,7 +354,7 @@ template handlePossibleMapStart(flow: bool = false,
if flow and p.lexer.buf[pos] notin {' ', '\t'}: if flow and p.lexer.buf[pos] notin {' ', '\t'}:
recentJsonStyle = p.lexer.buf[pos] in {']', '}', '\'', '"'} recentJsonStyle = p.lexer.buf[pos] in {']', '}', '\'', '"'}
pos.inc() pos.inc()
if level.indentation == -1: level.indentation = indentation if level.indentation == UnknownIndentation: level.indentation = indentation
template handleBlockItemStart() {.dirty.} = template handleBlockItemStart() {.dirty.} =
case level.kind case level.kind
@ -356,11 +369,10 @@ template handleBlockItemStart() {.dirty.} =
level.kind = fplMapKey level.kind = fplMapKey
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: indentation) level = FastParseLevel(kind: fplUnknown, indentation: indentation)
of fplScalar, fplSinglePairKey, fplSinglePairValue: assert(false) of fplScalar, fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
template handleFlowItemStart() {.dirty.} = template handleFlowItemStart() {.dirty.} =
if level.kind == fplUnknown and ancestry.len > 0 and if level.kind == fplUnknown and ancestry[ancestry.high].kind == fplSequence:
ancestry[ancestry.high].kind == fplSequence:
handlePossibleMapStart(true, true) handlePossibleMapStart(true, true)
template startToken() {.dirty.} = template startToken() {.dirty.} =
@ -496,8 +508,8 @@ template tagUri(lexer: BaseLexer, uri: var string) =
c = lexer.buf[lexer.bufpos] c = lexer.buf[lexer.bufpos]
else: lexerError(lexer, "Invalid tag uri") else: lexerError(lexer, "Invalid tag uri")
template directivesEnd(lexer: BaseLexer, template directivesEndMarker(lexer: BaseLexer,
token: var LexedPossibleDirectivesEnd) = token: var LexedPossibleDirectivesEnd) =
debug("lex: directivesEnd") debug("lex: directivesEnd")
var p = lexer.bufpos + 1 var p = lexer.bufpos + 1
case lexer.buf[p] case lexer.buf[p]
@ -511,7 +523,7 @@ template directivesEnd(lexer: BaseLexer,
of spaceOrLineEnd: token = lpdeSequenceItem of spaceOrLineEnd: token = lpdeSequenceItem
else: token = lpdeScalarContent else: token = lpdeScalarContent
template documentEnd(lexer: var BaseLexer, isDocumentEnd: var bool) = template documentEndMarker(lexer: var BaseLexer, isDocumentEnd: var bool) =
var p = lexer.bufpos + 1 var p = lexer.bufpos + 1
if lexer.buf[p] == '.': if lexer.buf[p] == '.':
p.inc() p.inc()
@ -832,13 +844,12 @@ template blockScalar(lexer: BaseLexer, content: var string,
chomp: ChompType = ctClip chomp: ChompType = ctClip
detectedIndent = false detectedIndent = false
recentLineMoreIndented = false recentLineMoreIndented = false
let parentIndent = if ancestry.len > 0: let parentIndent = ancestry[ancestry.high].indentation
ancestry[ancestry.high].indentation else: -1
case lexer.buf[lexer.bufpos] case lexer.buf[lexer.bufpos]
of '|': literal = true of '|': literal = true
of '>': literal = false of '>': literal = false
else: assert(false) else: debugFail()
while true: while true:
lexer.bufpos.inc() lexer.bufpos.inc()
@ -865,7 +876,7 @@ template blockScalar(lexer: BaseLexer, content: var string,
of EndOfFile: of EndOfFile:
lexerError(lexer, "Missing content of block scalar") lexerError(lexer, "Missing content of block scalar")
# TODO: is this correct? # TODO: is this correct?
else: assert(false) else: debugFail()
var newlines = 0 var newlines = 0
content = "" content = ""
block outer: block outer:
@ -889,7 +900,7 @@ template blockScalar(lexer: BaseLexer, content: var string,
if parentIndent == -1 and lexer.buf[lexer.bufpos] == '.': if parentIndent == -1 and lexer.buf[lexer.bufpos] == '.':
var isDocumentEnd: bool var isDocumentEnd: bool
startToken() startToken()
lexer.documentEnd(isDocumentEnd) lexer.documentEndMarker(isDocumentEnd)
if isDocumentEnd: if isDocumentEnd:
stateAfter = fpExpectDocEnd stateAfter = fpExpectDocEnd
break outer break outer
@ -1092,7 +1103,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of '-': of '-':
var token: LexedPossibleDirectivesEnd var token: LexedPossibleDirectivesEnd
startToken() startToken()
p.lexer.directivesEnd(token) p.lexer.directivesEndMarker(token)
yield startDocEvent() yield startDocEvent()
case token case token
of lpdeDirectivesEnd: of lpdeDirectivesEnd:
@ -1116,7 +1127,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of '-': of '-':
var token: LexedPossibleDirectivesEnd var token: LexedPossibleDirectivesEnd
startToken() startToken()
p.lexer.directivesEnd(token) p.lexer.directivesEndMarker(token)
case token case token
of lpdeDirectivesEnd: of lpdeDirectivesEnd:
p.lexer.bufpos.inc(3) p.lexer.bufpos.inc(3)
@ -1137,14 +1148,14 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
else: else:
ensureCorrectIndentation() ensureCorrectIndentation()
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
content = "" content = ""
p.lexer.plainScalar(content, cBlock) p.lexer.plainScalar(content, cBlock)
state = fpBlockAfterPlainScalar state = fpBlockAfterPlainScalar
of '.': of '.':
var isDocumentEnd: bool var isDocumentEnd: bool
startToken() startToken()
p.lexer.documentEnd(isDocumentEnd) p.lexer.documentEndMarker(isDocumentEnd)
if isDocumentEnd: if isDocumentEnd:
closeEverything() closeEverything()
p.lexer.bufpos.inc(3) p.lexer.bufpos.inc(3)
@ -1160,7 +1171,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
else: else:
ensureCorrectIndentation() ensureCorrectIndentation()
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
content = "" content = ""
p.lexer.plainScalar(content, cBlock) p.lexer.plainScalar(content, cBlock)
state = fpBlockAfterPlainScalar state = fpBlockAfterPlainScalar
@ -1289,18 +1300,18 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
yield scalarEvent("", yTagQuestionMark, yAnchorNone) yield scalarEvent("", yTagQuestionMark, yAnchorNone)
level.kind = fplMapValue level.kind = fplMapValue
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
of fplMapValue: of fplMapValue:
level.kind = fplMapValue level.kind = fplMapValue
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
of fplSequence: of fplSequence:
startToken() startToken()
parserError("Illegal token (expected sequence item)") parserError("Illegal token (expected sequence item)")
of fplScalar: of fplScalar:
startToken() startToken()
parserError("Multiline scalars may not be implicit map keys") parserError("Multiline scalars may not be implicit map keys")
of fplSinglePairKey, fplSinglePairValue: assert(false) of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail()
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
p.lexer.skipWhitespace() p.lexer.skipWhitespace()
indentation = p.lexer.getColNumber(p.lexer.bufpos) indentation = p.lexer.getColNumber(p.lexer.bufpos)
@ -1320,11 +1331,11 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of '\l': of '\l':
p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos)
state = fpBlockLineStart state = fpBlockLineStart
level.indentation = -1 level.indentation = UnknownIndentation
of '\c': of '\c':
p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos)
state = fpBlockLineStart state = fpBlockLineStart
level.indentation = -1 level.indentation = UnknownIndentation
of EndOfFile: of EndOfFile:
closeEverything() closeEverything()
return return
@ -1332,7 +1343,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
p.lexer.lineEnding() p.lexer.lineEnding()
handleLineEnd(true) handleLineEnd(true)
state = fpBlockLineStart state = fpBlockLineStart
level.indentation = -1 level.indentation = UnknownIndentation
of '\'': of '\'':
handleBlockItemStart() handleBlockItemStart()
content = "" content = ""
@ -1418,11 +1429,12 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
case p.lexer.buf[p.lexer.bufpos] case p.lexer.buf[p.lexer.bufpos]
of '-': of '-':
var token: LexedPossibleDirectivesEnd var token: LexedPossibleDirectivesEnd
p.lexer.directivesEnd(token) p.lexer.directivesEndMarker(token)
case token case token
of lpdeDirectivesEnd: of lpdeDirectivesEnd:
p.lexer.bufpos.inc(3) p.lexer.bufpos.inc(3)
yield endDocEvent() yield endDocEvent()
discard ancestry.pop()
initDocValues() initDocValues()
yield startDocEvent() yield startDocEvent()
state = fpBlockObjectStart state = fpBlockObjectStart
@ -1431,7 +1443,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of '.': of '.':
var isDocumentEnd: bool var isDocumentEnd: bool
startToken() startToken()
p.lexer.documentEnd(isDocumentEnd) p.lexer.documentEndMarker(isDocumentEnd)
if isDocumentEnd: if isDocumentEnd:
closeEverything() closeEverything()
p.lexer.bufpos.inc(3) p.lexer.bufpos.inc(3)
@ -1456,7 +1468,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of '.': of '.':
var isDocumentEnd: bool var isDocumentEnd: bool
startToken() startToken()
p.lexer.documentEnd(isDocumentEnd) p.lexer.documentEndMarker(isDocumentEnd)
if isDocumentEnd: if isDocumentEnd:
p.lexer.bufpos.inc(3) p.lexer.bufpos.inc(3)
p.lexer.lineEnding() p.lexer.lineEnding()
@ -1511,7 +1523,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of fplSinglePairValue: of fplSinglePairValue:
startToken() startToken()
parserError("Unexpected token (expected ']')") parserError("Unexpected token (expected ']')")
of fplUnknown, fplScalar, fplSinglePairKey: assert(false) of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
leaveFlowLevel() leaveFlowLevel()
of ']': of ']':
@ -1529,7 +1541,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of fplMapKey, fplMapValue: of fplMapKey, fplMapValue:
startToken() startToken()
parserError("Unexpected token (expected '}')") parserError("Unexpected token (expected '}')")
of fplUnknown, fplScalar, fplSinglePairKey: assert(false) of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
leaveFlowLevel() leaveFlowLevel()
of ',': of ',':
@ -1550,9 +1562,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
level = ancestry.pop() level = ancestry.pop()
yield endMapEvent() yield endMapEvent()
assert(level.kind == fplSequence) assert(level.kind == fplSequence)
of fplUnknown, fplScalar, fplSinglePairKey: assert(false) of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
of ':': of ':':
if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cFlow): if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cFlow):
@ -1563,13 +1575,15 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
case level.kind case level.kind
of fplSequence: of fplSequence:
yield startMapEvent(tag, anchor) yield startMapEvent(tag, anchor)
debug("started single-pair map at " & (if level.indentation == -1: debug("started single-pair map at " &
$indentation else: $level.indentation)) (if level.indentation == UnknownIndentation:
$indentation else: $level.indentation))
tag = yTagQuestionMark tag = yTagQuestionMark
anchor = yAnchorNone anchor = yAnchorNone
if level.indentation == -1: level.indentation = indentation if level.indentation == UnknownIndentation:
level.indentation = indentation
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplSinglePairValue, indentation: -1) level = initLevel(fplSinglePairValue)
yield scalarEvent("") yield scalarEvent("")
of fplMapValue, fplSinglePairValue: of fplMapValue, fplSinglePairValue:
startToken() startToken()
@ -1580,10 +1594,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of fplSinglePairKey: of fplSinglePairKey:
yieldEmptyScalar() yieldEmptyScalar()
level.kind = fplSinglePairValue level.kind = fplSinglePairValue
of fplUnknown, fplScalar: of fplUnknown, fplScalar, fplDocument: debugFail()
assert(false)
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
of '\'': of '\'':
handleFlowItemStart() handleFlowItemStart()
@ -1618,13 +1631,12 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
startToken() startToken()
parserError("Duplicate '?' in flow mapping") parserError("Duplicate '?' in flow mapping")
elif level.kind == fplUnknown: elif level.kind == fplUnknown:
if ancestry.len > 0: case ancestry[ancestry.high].kind
case ancestry[ancestry.high].kind of fplMapKey, fplMapValue, fplDocument: discard
of fplMapKey, fplMapValue: discard of fplSequence: handleObjectStart(yamlStartMap, true)
of fplSequence: handleObjectStart(yamlStartMap, true) else:
else: startToken()
startToken() parserError("Unexpected token")
parserError("Unexpected token")
explicitFlowKey = true explicitFlowKey = true
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
else: else:
@ -1647,7 +1659,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
level = ancestry.pop() level = ancestry.pop()
assert(level.kind == fplSequence) assert(level.kind == fplSequence)
yield endMapEvent() yield endMapEvent()
of fplScalar, fplUnknown, fplSinglePairKey: assert(false) of fplScalar, fplUnknown, fplSinglePairKey, fplDocument: debugFail()
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
leaveFlowLevel() leaveFlowLevel()
of '}': of '}':
@ -1656,7 +1668,7 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
of fplSequence, fplSinglePairValue: of fplSequence, fplSinglePairValue:
startToken() startToken()
parserError("Unexpected token (expected ']')") parserError("Unexpected token (expected ']')")
of fplUnknown, fplScalar, fplSinglePairKey: assert(false) of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
leaveFlowLevel() leaveFlowLevel()
of ',': of ',':
@ -1671,9 +1683,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
assert(level.kind == fplSequence) assert(level.kind == fplSequence)
yield endMapEvent() yield endMapEvent()
of fplMapKey: explicitFlowKey = false of fplMapKey: explicitFlowKey = false
of fplUnknown, fplScalar, fplSinglePairKey: assert(false) of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
state = fpFlow state = fpFlow
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
of ':': of ':':
@ -1682,9 +1694,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
startToken() startToken()
parserError("Unexpected token (expected ',')") parserError("Unexpected token (expected ',')")
of fplMapValue, fplSinglePairValue: discard of fplMapValue, fplSinglePairValue: discard
of fplUnknown, fplScalar, fplSinglePairKey: assert(false) of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail()
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = initLevel(fplUnknown)
state = fpFlow state = fpFlow
p.lexer.bufpos.inc() p.lexer.bufpos.inc()
of '#': of '#':
@ -1697,4 +1709,4 @@ proc parse*(p: YamlParser, s: Stream): YamlStream =
startToken() startToken()
parserError("Unexpected content (expected flow indicator)") parserError("Unexpected content (expected flow indicator)")
try: result = initYamlStream(backend) try: result = initYamlStream(backend)
except Exception: assert(false) # compiler error except Exception: debugFail() # compiler error

4
test/yamlTestSuite.nim
View File

@ -37,8 +37,8 @@ for kind, dirPath in walkDir("yaml-dev-kit"):
expected = parseEventStream(expectedIn, tagLib) expected = parseEventStream(expectedIn, tagLib)
styledWriteLine(stdout, fgBlue, "[test] ", fgWhite, dirPath[^4..^1], styledWriteLine(stdout, fgBlue, "[test] ", fgWhite, dirPath[^4..^1],
": ", strip(readFile(dirPath / "===")), resetStyle) ": ", strip(readFile(dirPath / "===")), resetStyle)
var i = 1
try: try:
var i = 1
while not actual.finished(): while not actual.finished():
let actualEvent = actual.next() let actualEvent = actual.next()
if expected.finished(): if expected.finished():
@ -73,6 +73,8 @@ for kind, dirPath in walkDir("yaml-dev-kit"):
pe.msg pe.msg
echo pe.lineContent echo pe.lineContent
else: echo e.msg else: echo e.msg
echoError("Catched an exception at token #" & $i &
" test was not successful")
actualIn.close() actualIn.close()
expectedIn.close() expectedIn.close()