status-im/NimYAML
2
0
Fork 0
mirror of https://github.com/status-im/NimYAML.git synced 2025-01-12 04:24:16 +00:00
Code Issues Projects Releases Wiki Activity

fastparse: Support flow object as implicit keys

Browse Source
This commit is contained in:
Felix Krause 2016-01-22 21:36:11 +01:00
parent 7af566c3a0
commit f670c5c86b
1 changed files with 222 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

419
private/fastparse.nim
View File

@ -1,6 +1,6 @@
type type
FastParseState = enum FastParseState = enum
fpInitial, fpBlockLineStart, fpBlockAfterScalar, fpBlockAfterPlainScalar, fpInitial, fpBlockLineStart, fpBlockAfterObject, fpBlockAfterPlainScalar,
fpBlockObjectStart, fpBlockContinueScalar, fpExpectDocEnd, fpFlow, fpBlockObjectStart, fpBlockContinueScalar, fpExpectDocEnd, fpFlow,
fpFlowAfterObject fpFlowAfterObject
@ -51,7 +51,7 @@ template raiseError(message: string, col: int) {.dirty.} =
repeat(' ', lexer.getColNumber(lexer.bufpos)) & "^\n" repeat(' ', lexer.getColNumber(lexer.bufpos)) & "^\n"
raise e raise e
template closeLevel() {.dirty.} = template yieldLevelEnd() {.dirty.} =
case level.kind case level.kind
of fplSequence: of fplSequence:
yield endSeqEvent() yield endSeqEvent()
@ -64,11 +64,9 @@ template closeLevel() {.dirty.} =
yield endMapEvent() yield endMapEvent()
of fplScalar: of fplScalar:
applyObjectProperties() applyObjectProperties()
yield cachedScalar yield scalarEvent(content, tag, anchor)
of fplUnknown: of fplUnknown:
yield scalarEvent("") yield scalarEvent("")
if ancestry.len > 0:
level = ancestry.pop()
template handleLineEnd(insideDocument: bool) {.dirty.} = template handleLineEnd(insideDocument: bool) {.dirty.} =
case lexer.buf[lexer.bufpos] case lexer.buf[lexer.bufpos]
@ -83,70 +81,72 @@ template handleLineEnd(insideDocument: bool) {.dirty.} =
else: else:
discard discard
template handleObjectEnd() {.dirty.} = template handleObjectEnd(nextState: FastParseState) {.dirty.} =
tag = yTagQuestionMark if ancestry.len == 0:
anchor = yAnchorNone state = fpExpectDocEnd
case level.kind else:
of fplMapKey: level = ancestry.pop()
level.kind = fplMapValue state = nextState
of fplMapValue: tag = yTagQuestionMark
level.kind = fplMapKey anchor = yAnchorNone
of fplSequence: case level.kind
discard of fplMapKey:
of fplUnknown, fplScalar: level.kind = fplMapValue
raiseError("Internal error!") of fplMapValue:
level.kind = fplMapKey
of fplSequence:
discard
of fplUnknown, fplScalar:
assert(false)
template handleStartObject(k: YamlStreamEventKind) {.dirty.} = template handleObjectStart(k: YamlStreamEventKind) {.dirty.} =
assert(level.kind == fplUnknown)
when k == yamlStartMap: when k == yamlStartMap:
yield startMapEvent(objectTag, objectAnchor) yield startMapEvent(objectTag, objectAnchor)
debug("started map at " & $lexer.tokenstart) debug("started map at " & $indentation)
level.kind = fplMapKey
else: else:
yield startSeqEvent(objectTag, objectAnchor) yield startSeqEvent(objectTag, objectAnchor)
debug("started sequence at " & $lexer.tokenstart) debug("started sequence at " & $indentation)
level.kind = fplSequence
objectTag = yTagQuestionMark objectTag = yTagQuestionMark
objectAnchor = yAnchorNone objectAnchor = yAnchorNone
level.indentation = indentation
ancestry.add(level)
level.kind = fplUnknown
template closeMoreIndentedLevels() {.dirty.} = template closeMoreIndentedLevels() {.dirty.} =
while ancestry.len > 0: while ancestry.len > 0:
let parent = ancestry[ancestry.high] let parent = ancestry[ancestry.high]
if parent.indentation >= indentation: if parent.indentation >= indentation:
debug("Closing because level.indentation =" & $level.indentation & debug("Closing because parent.indentation (" & $parent.indentation &
", but indentation = " & $indentation) ") >= indentation(" & $indentation & ")")
closeLevel() yieldLevelEnd()
handleObjectEnd() handleObjectEnd(fpBlockAfterObject)
else: else:
break break
template closeEverything() {.dirty.} = template closeEverything() {.dirty.} =
indentation = 0 indentation = 0
closeMoreIndentedLevels() closeMoreIndentedLevels()
closeLevel() yieldLevelEnd()
yield endDocEvent() yield endDocEvent()
template handleStartBlockSequence() {.dirty.} = template handleBlockSequenceIndicator() {.dirty.} =
case level.kind case level.kind
of fplUnknown: of fplUnknown:
level.kind = fplSequence handleObjectStart(yamlStartSequence)
handleStartObject(yamlStartSequence)
of fplSequence: of fplSequence:
if level.indentation != indentation: if level.indentation != indentation:
raiseError("Invalid indentation of block sequence indicator", raiseError("Invalid indentation of block sequence indicator",
lexer.bufpos) lexer.bufpos)
ancestry.add(level)
level.kind = fplUnknown
else: else:
raiseError("Illegal sequence item in map") raiseError("Illegal sequence item in map")
ancestry.add(level)
lexer.skipWhitespace() lexer.skipWhitespace()
indentation = lexer.getColNumber(lexer.bufpos) indentation = lexer.getColNumber(lexer.bufpos)
level = FastParseLevel(kind: fplUnknown, indentation: indentation) level.indentation = indentation
template handleStartBlockScalar() {.dirty.} =
case level.kind
of fplUnknown, fplMapKey:
discard
of fplSequence:
raiseError("Illegal token (expected '- ')")
of fplMapValue, fplScalar:
raiseError("Internal error!")
template propsToObjectProps() {.dirty.} = template propsToObjectProps() {.dirty.} =
if objectTag == yTagQuestionmark: if objectTag == yTagQuestionmark:
@ -166,7 +166,7 @@ template initDocValues() {.dirty.} =
shorthands["!"] = "!" shorthands["!"] = "!"
shorthands["!!"] = "tag:yaml.org,2002:" shorthands["!!"] = "tag:yaml.org,2002:"
nextAnchorId = 0.AnchorId nextAnchorId = 0.AnchorId
level = FastParseLevel(kind: fplUnknown, indentation: -1) level = FastParseLevel(kind: fplUnknown, indentation: 0)
tag = yTagQuestionmark tag = yTagQuestionmark
objectTag = yTagQuestionmark objectTag = yTagQuestionmark
anchor = yAnchorNone anchor = yAnchorNone
@ -174,20 +174,22 @@ template initDocValues() {.dirty.} =
template applyObjectProperties() {.dirty.} = template applyObjectProperties() {.dirty.} =
if objectTag != yTagQuestionmark: if objectTag != yTagQuestionmark:
if cachedScalar.scalarTag != yTagQuestionmark: if tag != yTagQuestionmark:
debug("cached = " & $cachedScalar.scalarTag & ", object = " & $objectTag) debug("tag = " & $tag & ", object = " & $objectTag)
raiseError("Only one tag is allowed per node") raiseError("Only one tag is allowed per node")
else: else:
cachedScalar.scalarTag = objectTag tag = objectTag
objectTag = yTagQuestionmark objectTag = yTagQuestionmark
if objectAnchor != yAnchorNone: if objectAnchor != yAnchorNone:
if cachedScalar.scalarAnchor != yAnchorNone: if anchor != yAnchorNone:
raiseError("Only one anchor is allowed per node") raiseError("Only one anchor is allowed per node")
else: else:
cachedScalar.scalarAnchor = objectAnchor anchor = objectAnchor
objectAnchor = yAnchorNone objectAnchor = yAnchorNone
template handleTagHandle() {.dirty.} = template handleTagHandle() {.dirty.} =
if level.kind != fplUnknown:
raiseError("Unexpected token", lexer.bufpos)
if tag != yTagQuestionmark: if tag != yTagQuestionmark:
raiseError("Only one tag handle is allowed per node") raiseError("Only one tag handle is allowed per node")
content = "" content = ""
@ -209,6 +211,8 @@ template handleTagHandle() {.dirty.} =
tag = tagLib.registerUri(tagUri) tag = tagLib.registerUri(tagUri)
template handleAnchor() {.dirty.} = template handleAnchor() {.dirty.} =
if level.kind != fplUnknown:
raiseError("Unexpected token", lexer.bufpos)
if anchor != yAnchorNone: if anchor != yAnchorNone:
raiseError("Only one anchor is allowed per node", lexer.bufpos) raiseError("Only one anchor is allowed per node", lexer.bufpos)
content = "" content = ""
@ -218,28 +222,71 @@ template handleAnchor() {.dirty.} =
nextAnchorId = cast[AnchorId](cast[int](nextAnchorId) + 1) nextAnchorId = cast[AnchorId](cast[int](nextAnchorId) + 1)
template handleAlias() {.dirty.} = template handleAlias() {.dirty.} =
if level.kind != fplUnknown:
raiseError("Unexpected token", lexer.bufpos)
if anchor != yAnchorNone or tag != yTagQuestionmark: if anchor != yAnchorNone or tag != yTagQuestionmark:
raiseError("Alias may not have anchor or tag") raiseError("Alias may not have anchor or tag")
content = "" content = ""
lexer.anchorName(content) lexer.anchorName(content)
var id: AnchorId
try: try:
cachedScalar = aliasEvent(anchors[content]) id = anchors[content]
except KeyError: except KeyError:
raiseError("Unknown anchor") raiseError("Unknown anchor")
state = fpBlockAfterScalar yield aliasEvent(id)
handleObjectEnd(fpBlockAfterObject)
template leaveFlowLevel() {.dirty.} = template leaveFlowLevel() {.dirty.} =
flowdepth.inc(-1) flowdepth.inc(-1)
if ancestry.len == 0: if flowdepth == 0:
state = fpExpectDocEnd yieldLevelEnd()
handleObjectEnd(fpBlockAfterObject)
else: else:
level = ancestry.pop() yieldLevelEnd()
if flowdepth == 0: handleObjectEnd(fpFlowAfterObject)
lexer.lineEnding()
handleLineEnd(true) template handlePossibleMapStart() {.dirty.} =
state = fpBlockLineStart var flowDepth = 0
for p in countup(lexer.bufpos, lexer.bufpos + 1024):
case lexer.buf[p]
of ':':
if flowDepth == 0 and lexer.buf[p + 1] in spaceOrLineEnd:
handleObjectStart(yamlStartMap)
break
else:
echo "found : but ignoring (flowDepth=", flowDepth, ", next=", lexer.buf[p + 1]
of lineEnd:
break
of '[', '{':
flowDepth.inc()
of '}', ']':
flowDepth.inc(-1)
of '?':
if flowDepth == 0: break
of '#':
if lexer.buf[p - 1] in space: break
else: else:
state = fpFlowAfterObject discard
template handleBlockItemStart() {.dirty.} =
case level.kind
of fplUnknown:
discard
of fplSequence:
raiseError("Unexpected token (expected block sequence indicator)",
lexer.bufpos)
of fplMapKey:
ancestry.add(level)
level.kind = fplUnknown
of fplMapValue:
yield scalarEvent("", tag, anchor)
tag = yTagQuestionmark
anchor = yAnchorNone
level.kind = fplMapKey
ancestry.add(level)
level.kind = fplUnknown
of fplScalar:
assert(false)
template finishLine(lexer: FastLexer) = template finishLine(lexer: FastLexer) =
debug("lex: finishLine") debug("lex: finishLine")
@ -375,20 +422,17 @@ template tagUri(lexer: FastLexer, uri: var string) =
else: else:
raiseError("Invalid tag uri") raiseError("Invalid tag uri")
template directivesEnd(lexer: FastLexer, content: var string, template directivesEnd(lexer: FastLexer,
token: var LexedPossibleDirectivesEnd) = token: var LexedPossibleDirectivesEnd) =
debug("lex: directivesEnd") debug("lex: directivesEnd")
content.add('-')
lexer.tokenstart = lexer.getColNumber(lexer.bufpos) lexer.tokenstart = lexer.getColNumber(lexer.bufpos)
lexer.bufpos.inc() var p = lexer.bufpos + 1
case lexer.buf[lexer.bufpos] case lexer.buf[p]
of '-': of '-':
content.add('-') p.inc()
lexer.bufpos.inc() if lexer.buf[p] == '-':
if lexer.buf[lexer.bufpos] == '-': p.inc()
content.add('-') if lexer.buf[p] in spaceOrLineEnd:
lexer.bufpos.inc()
if lexer.buf[lexer.bufpos] in spaceOrLineEnd:
token = lpdeDirectivesEnd token = lpdeDirectivesEnd
else: else:
token = lpdeScalarContent token = lpdeScalarContent
@ -399,18 +443,14 @@ template directivesEnd(lexer: FastLexer, content: var string,
else: else:
token = lpdeScalarContent token = lpdeScalarContent
template documentEnd(lexer: var FastLexer, content: var string, template documentEnd(lexer: var FastLexer, isDocumentEnd: var bool) =
isDocumentEnd: var bool) =
content.add('.')
lexer.tokenstart = lexer.getColNumber(lexer.bufpos) lexer.tokenstart = lexer.getColNumber(lexer.bufpos)
lexer.bufpos.inc() var p = lexer.bufpos + 1
if lexer.buf[lexer.bufpos] == '.': if lexer.buf[p] == '.':
content.add('.') p.inc()
lexer.bufpos.inc() if lexer.buf[p] == '.':
if lexer.buf[lexer.bufpos] == '.': p.inc()
content.add('.') if lexer.buf[p] in spaceOrLineEnd:
lexer.bufpos.inc()
if lexer.buf[lexer.bufpos] in spaceOrLineEnd:
isDocumentEnd = true isDocumentEnd = true
else: else:
isDocumentEnd = false isDocumentEnd = false
@ -563,9 +603,8 @@ template plainScalar(lexer: FastLexer, content: var string,
content.add(c) content.add(c)
template continueMultilineScalar() {.dirty.} = template continueMultilineScalar() {.dirty.} =
cachedScalar.scalarContent.add(if newlines == 1: " " else: content.add(if newlines == 1: " " else: repeat('\x0A', newlines - 1))
repeat('\x0A', newlines - 1)) lexer.plainScalar(content, cBlockOut)
lexer.plainScalar(cachedScalar.scalarContent, cBlockOut)
state = fpBlockAfterPlainScalar state = fpBlockAfterPlainScalar
template handleFlowPlainScalar() {.dirty.} = template handleFlowPlainScalar() {.dirty.} =
@ -600,10 +639,12 @@ template handleFlowPlainScalar() {.dirty.} =
newlines = 0 newlines = 0
lexer.plainScalar(content, cFlowOut) lexer.plainScalar(content, cFlowOut)
yield scalarEvent(content, tag, anchor) yield scalarEvent(content, tag, anchor)
tag = yTagQuestionMark handleObjectEnd(fpFlowAfterObject)
anchor = yAnchorNone
level = ancestry.pop() template ensureCorrectIndentation() {.dirty.} =
state = fpFlowAfterObject if level.indentation != indentation:
raiseError("Invalid indentation (expected indentation for " & $level.kind &
" :" & $level.indentation & ")", lexer.bufpos)
template tagHandle(lexer: var FastLexer, content: var string, template tagHandle(lexer: var FastLexer, content: var string,
shorthandEnd: var int) = shorthandEnd: var int) =
@ -675,7 +716,6 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
anchor, objectAnchor: AnchorId anchor, objectAnchor: AnchorId
ancestry = newSeq[FastParseLevel]() ancestry = newSeq[FastParseLevel]()
level: FastParseLevel level: FastParseLevel
cachedScalar: YamlStreamEvent
indentation: int indentation: int
newlines: int newlines: int
flowdepth: int = 0 flowdepth: int = 0
@ -726,19 +766,20 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
handleLineEnd(false) handleLineEnd(false)
of '-': of '-':
var token: LexedPossibleDirectivesEnd var token: LexedPossibleDirectivesEnd
content = "" lexer.directivesEnd(token)
lexer.directivesEnd(content, token)
yield startDocEvent() yield startDocEvent()
case token case token
of lpdeDirectivesEnd: of lpdeDirectivesEnd:
lexer.bufpos.inc(3)
state = fpBlockObjectStart state = fpBlockObjectStart
of lpdeSequenceItem: of lpdeSequenceItem:
indentation = 0 indentation = 0
handleStartBlockSequence() lexer.bufpos.inc()
handleBlockSequenceIndicator()
state = fpBlockObjectStart state = fpBlockObjectStart
of lpdeScalarContent: of lpdeScalarContent:
content = ""
lexer.plainScalar(content, cBlockOut) lexer.plainScalar(content, cBlockOut)
cachedScalar = scalarEvent(content, tag, anchor)
state = fpBlockAfterPlainScalar state = fpBlockAfterPlainScalar
else: else:
yield startDocEvent() yield startDocEvent()
@ -748,10 +789,10 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
case lexer.buf[lexer.bufpos] case lexer.buf[lexer.bufpos]
of '-': of '-':
var token: LexedPossibleDirectivesEnd var token: LexedPossibleDirectivesEnd
content = "" lexer.directivesEnd(token)
lexer.directivesEnd(content, token)
case token case token
of lpdeDirectivesEnd: of lpdeDirectivesEnd:
lexer.bufpos.inc(3)
closeEverything() closeEverything()
initDocValues() initDocValues()
yield startDocEvent() yield startDocEvent()
@ -759,30 +800,47 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
of lpdeSequenceItem: of lpdeSequenceItem:
indentation = 0 indentation = 0
closeMoreIndentedLevels() closeMoreIndentedLevels()
handleStartBlockSequence() lexer.bufpos.inc()
handleBlockSequenceIndicator()
state = fpBlockObjectStart state = fpBlockObjectStart
of lpdeScalarContent: of lpdeScalarContent:
if level.kind == fplScalar: case level.kind
of fplScalar:
continueMultilineScalar() continueMultilineScalar()
of fplUnknown:
handlePossibleMapStart()
else: else:
ensureCorrectIndentation()
ancestry.add(level)
level.kind = fplUnknown
content = ""
lexer.plainScalar(content, cBlockOut) lexer.plainScalar(content, cBlockOut)
cachedScalar = scalarEvent(content, tag, anchor)
state = fpBlockAfterPlainScalar state = fpBlockAfterPlainScalar
of '.': of '.':
var isDocumentEnd: bool var isDocumentEnd: bool
content = "" lexer.documentEnd(isDocumentEnd)
lexer.documentEnd(content, isDocumentEnd)
if isDocumentEnd: if isDocumentEnd:
lexer.bufpos.inc(3)
lexer.lineEnding() lexer.lineEnding()
handleLineEnd(true)
closeEverything() closeEverything()
initDocValues() initDocValues()
state = fpInitial state = fpInitial
elif level.kind == fplScalar:
continueMultilineScalar()
else: else:
lexer.plainScalar(content, cBlockOut) indentation = 0
cachedScalar = scalarEvent(content, tag, anchor) closeMoreIndentedLevels()
state = fpBlockAfterPlainScalar case level.kind
of fplUnknown:
handlePossibleMapStart()
of fplScalar:
continueMultilineScalar()
else:
ensureCorrectIndentation()
ancestry.add(level)
level.kind = fplUnknown
content = ""
lexer.plainScalar(content, cBlockOut)
state = fpBlockAfterPlainScalar
of ' ': of ' ':
lexer.skipIndentation() lexer.skipIndentation()
indentation = lexer.getColNumber(lexer.bufpos) indentation = lexer.getColNumber(lexer.bufpos)
@ -791,9 +849,11 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
of fplScalar: of fplScalar:
state = fpBlockContinueScalar state = fpBlockContinueScalar
of fplUnknown: of fplUnknown:
handlePossibleMapStart()
state = fpBlockObjectStart state = fpBlockObjectStart
level.indentation = indentation level.indentation = indentation
else: else:
ensureCorrectIndentation()
state = fpBlockObjectStart state = fpBlockObjectStart
else: else:
indentation = 0 indentation = 0
@ -802,9 +862,11 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
of fplScalar: of fplScalar:
state = fpBlockContinueScalar state = fpBlockContinueScalar
of fplUnknown: of fplUnknown:
handlePossibleMapStart()
state = fpBlockObjectStart state = fpBlockObjectStart
level.indentation = indentation level.indentation = indentation
else: else:
ensureCorrectIndentation()
state = fpBlockObjectStart state = fpBlockObjectStart
of fpBlockContinueScalar: of fpBlockContinueScalar:
debug("state: blockAfterPlainScalar") debug("state: blockAfterPlainScalar")
@ -823,18 +885,12 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
else: else:
raiseError("Unexpected token", lexer.bufpos) raiseError("Unexpected token", lexer.bufpos)
of '#': of '#':
yield cachedScalar yield scalarEvent(content, tag, anchor)
lexer.lineEnding() lexer.lineEnding()
handleLineEnd(true) handleLineEnd(true)
if ancestry.len == 0: handleObjectEnd(fpBlockLineStart)
state = fpExpectDocEnd
else:
level = ancestry.pop()
handleObjectEnd()
state = fpBlockLineStart
else: else:
continueMultilineScalar() continueMultilineScalar()
of fpBlockAfterPlainScalar: of fpBlockAfterPlainScalar:
debug("state: blockAfterPlainScalar") debug("state: blockAfterPlainScalar")
lexer.skipWhitespace() lexer.skipWhitespace()
@ -854,74 +910,55 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
lexer.bufpos = lexer.handleCR(lexer.bufpos) lexer.bufpos = lexer.handleCR(lexer.bufpos)
state = fpBlockLineStart state = fpBlockLineStart
else: else:
state = fpBlockAfterScalar yield scalarEvent(content, tag, anchor)
of fpBlockAfterScalar: handleObjectEnd(fpBlockAfterObject)
debug("state: blockAfterScalar") of fpBlockAfterObject:
debug("state: blockAfterObject")
lexer.skipWhitespace() lexer.skipWhitespace()
case lexer.buf[lexer.bufpos] case lexer.buf[lexer.bufpos]
of EndOfFile: of EndOfFile:
level.kind = fplScalar
closeEverything() closeEverything()
break break
of '\x0A': of '\x0A':
if level.kind != fplUnknown: state = fpBlockLineStart
raiseError("Unexpected scalar")
applyObjectProperties()
yield cachedScalar
if ancestry.len == 0:
state = fpExpectDocEnd
else:
level = ancestry.pop()
handleObjectEnd()
state = fpBlockLineStart
lexer.bufpos = lexer.handleLF(lexer.bufpos) lexer.bufpos = lexer.handleLF(lexer.bufpos)
of '\c': of '\c':
if level.kind != fplUnknown: state = fpBlockLineStart
raiseError("Unexpected scalar")
applyObjectProperties()
yield cachedScalar
if ancestry.len == 0:
state = fpExpectDocEnd
else:
level = ancestry.pop()
handleObjectEnd()
state = fpBlockLineStart
lexer.bufpos = lexer.handleCR(lexer.bufpos) lexer.bufpos = lexer.handleCR(lexer.bufpos)
of ':': of ':':
case level.kind case level.kind
of fplUnknown: of fplUnknown:
level.kind = fplMapKey handleObjectStart(yamlStartMap)
handleStartObject(yamlStartMap)
of fplMapValue:
yield scalarEvent("", yTagQuestionMark, yAnchorNone)
level.kind = fplMapKey
of fplMapKey: of fplMapKey:
if level.indentation != indentation: yield scalarEvent("", yTagQuestionMark, yAnchorNone)
raiseError("Invalid indentation for map key") level.kind = fplMapValue
ancestry.add(level)
level.kind = fplUnknown
of fplMapValue:
level.kind = fplMapValue
ancestry.add(level)
level.kind = fplUnknown
of fplSequence: of fplSequence:
raiseError("Illegal token (expected sequence item)") raiseError("Illegal token (expected sequence item)")
of fplScalar: of fplScalar:
raiseError("Multiline scalars may not be implicit map keys") raiseError("Multiline scalars may not be implicit map keys")
handleObjectEnd()
yield cachedScalar
ancestry.add(level)
lexer.bufpos.inc() lexer.bufpos.inc()
lexer.skipWhitespace() lexer.skipWhitespace()
indentation = lexer.getColNumber(lexer.bufpos) indentation = lexer.getColNumber(lexer.bufpos)
level = FastParseLevel(kind: fplUnknown, indentation: indentation) level.indentation = indentation
state = fpBlockObjectStart state = fpBlockObjectStart
of '#': of '#':
applyObjectProperties() applyObjectProperties()
yield cachedScalar
lexer.lineEnding() lexer.lineEnding()
handleLineEnd(true) handleLineEnd(true)
state = fpBlockLineStart handleObjectEnd(fpBlockLineStart)
else: else:
raiseError("Illegal token (expected ':', comment or line end)", raiseError("Illegal token (expected ':', comment or line end)",
lexer.bufpos) lexer.bufpos)
of fpBlockObjectStart: of fpBlockObjectStart:
debug("state: blockObjectStart") debug("state: blockObjectStart")
lexer.skipWhitespace() lexer.skipWhitespace()
indentation = lexer.getColNumber(lexer.bufpos)
let objectStart = lexer.getColNumber(lexer.bufpos) let objectStart = lexer.getColNumber(lexer.bufpos)
case lexer.buf[lexer.bufpos] case lexer.buf[lexer.bufpos]
of '\x0A': of '\x0A':
@ -939,54 +976,56 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
lexer.lineEnding() lexer.lineEnding()
handleLineEnd(true) handleLineEnd(true)
of '\'': of '\'':
handleStartBlockScalar() handleBlockItemStart()
content = "" content = ""
lexer.singleQuotedScalar(content) lexer.singleQuotedScalar(content)
if tag == yTagQuestionMark: if tag == yTagQuestionMark:
tag = yTagExclamationMark tag = yTagExclamationMark
cachedScalar = scalarEvent(content, tag, anchor) yield scalarEvent(content, tag, anchor)
state = fpBlockAfterScalar handleObjectEnd(fpBlockAfterObject)
of '"': of '"':
handleStartBlockScalar() handleBlockItemStart()
content = "" content = ""
lexer.doublyQuotedScalar(content) lexer.doublyQuotedScalar(content)
if tag == yTagQuestionMark: if tag == yTagQuestionMark:
tag = yTagExclamationMark tag = yTagExclamationMark
cachedScalar = scalarEvent(content, tag, anchor) yield scalarEvent(content, tag, anchor)
state = fpBlockAfterScalar handleObjectEnd(fpBlockAfterObject)
of '-': of '-':
if lexer.isPlainSafe(lexer.bufpos + 1, cBlockOut): if lexer.isPlainSafe(lexer.bufpos + 1, cBlockOut):
handleStartBlockScalar() handleBlockItemStart()
lexer.tokenstart = lexer.getColNumber(lexer.bufpos) lexer.tokenstart = lexer.getColNumber(lexer.bufpos)
lexer.plainScalar(content, cBlockOut) lexer.plainScalar(content, cBlockOut)
cachedScalar = scalarEvent(content, tag, anchor)
state = fpBlockAfterPlainScalar state = fpBlockAfterPlainScalar
else: else:
lexer.bufpos.inc() lexer.bufpos.inc()
handleStartBlockSequence() handleBlockSequenceIndicator()
of '!': of '!':
handleBlockItemStart()
handleTagHandle() handleTagHandle()
of '&': of '&':
handleBlockItemStart()
handleAnchor() handleAnchor()
of '*': of '*':
handleBlockItemStart()
handleAlias() handleAlias()
of '[', '{': of '[', '{':
handleBlockItemStart()
applyObjectProperties() applyObjectProperties()
state = fpFlow state = fpFlow
else: else:
handleStartBlockScalar() handleBlockItemStart()
content = "" content = ""
lexer.plainScalar(content, cBlockOut) lexer.plainScalar(content, cBlockOut)
cachedScalar = scalarEvent(content, tag, anchor)
state = fpBlockAfterPlainScalar state = fpBlockAfterPlainScalar
of fpExpectDocEnd: of fpExpectDocEnd:
case lexer.buf[lexer.bufpos] case lexer.buf[lexer.bufpos]
of '-': of '-':
var token: LexedPossibleDirectivesEnd var token: LexedPossibleDirectivesEnd
content = "" lexer.directivesEnd(token)
lexer.directivesEnd(content, token)
case token case token
of lpdeDirectivesEnd: of lpdeDirectivesEnd:
lexer.bufpos.inc(3)
yield endDocEvent() yield endDocEvent()
initDocValues() initDocValues()
yield startDocEvent() yield startDocEvent()
@ -995,9 +1034,9 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
raiseError("Unexpected content (expected document end)") raiseError("Unexpected content (expected document end)")
of '.': of '.':
var isDocumentEnd: bool var isDocumentEnd: bool
content = "" lexer.documentEnd(isDocumentEnd)
lexer.documentEnd(content, isDocumentEnd)
if isDocumentEnd: if isDocumentEnd:
lexer.bufpos.inc(3)
yield endDocEvent() yield endDocEvent()
initDocValues() initDocValues()
state = fpInitial state = fpInitial
@ -1016,27 +1055,16 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
else: else:
raiseError("Unexpected content (expected document end)") raiseError("Unexpected content (expected document end)")
of fpFlow: of fpFlow:
debug("state: flow")
lexer.skipWhitespaceAndNewlines() lexer.skipWhitespaceAndNewlines()
case lexer.buf[lexer.bufpos] case lexer.buf[lexer.bufpos]
of '{': of '{':
assert(level.kind == fplUnknown) handleObjectStart(yamlStartMap)
yield startMapEvent(tag, anchor)
tag = yTagQuestionmark
anchor = yAnchorNone
flowdepth.inc() flowdepth.inc()
level.kind = fplMapKey
ancestry.add(level)
level = FastParseLevel(kind: fplUnknown)
lexer.bufpos.inc() lexer.bufpos.inc()
of '[': of '[':
assert(level.kind == fplUnknown) handleObjectStart(yamlStartSequence)
yield startSeqEvent(tag, anchor)
tag = yTagQuestionmark
anchor = yAnchorNone
flowdepth.inc() flowdepth.inc()
level.kind = fplSequence
ancestry.add(level)
level = FastParseLevel(kind: fplUnknown)
lexer.bufpos.inc() lexer.bufpos.inc()
of '}': of '}':
assert(level.kind == fplUnknown) assert(level.kind == fplUnknown)
@ -1046,15 +1074,15 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
yield scalarEvent("", tag, anchor) yield scalarEvent("", tag, anchor)
tag = yTagQuestionmark tag = yTagQuestionmark
anchor = yAnchorNone anchor = yAnchorNone
level.kind = fplMapKey
of fplMapKey: of fplMapKey:
discard discard
of fplSequence: of fplSequence:
raiseError("Unexpected token (expected ']')", lexer.bufpos) raiseError("Unexpected token (expected ']')", lexer.bufpos)
of fplUnknown, fplScalar: of fplUnknown, fplScalar:
assert(false) assert(false)
yield endMapEvent()
leaveFlowLevel()
lexer.bufpos.inc() lexer.bufpos.inc()
leaveFlowLevel()
of ']': of ']':
assert(level.kind == fplUnknown) assert(level.kind == fplUnknown)
level = ancestry.pop() level = ancestry.pop()
@ -1065,8 +1093,8 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
raiseError("Unexpected token (expected '}')", lexer.bufpos) raiseError("Unexpected token (expected '}')", lexer.bufpos)
of fplUnknown, fplScalar: of fplUnknown, fplScalar:
assert(false) assert(false)
leaveFlowLevel()
lexer.bufpos.inc() lexer.bufpos.inc()
leaveFlowLevel()
of ',': of ',':
assert(level.kind == fplUnknown) assert(level.kind == fplUnknown)
level = ancestry.pop() level = ancestry.pop()
@ -1115,84 +1143,79 @@ proc fastparse*(tagLib: TagLibrary, s: Stream): YamlStream =
if tag == yTagQuestionMark: if tag == yTagQuestionMark:
tag = yTagExclamationMark tag = yTagExclamationMark
yield scalarEvent(content, tag, anchor) yield scalarEvent(content, tag, anchor)
tag = yTagQuestionmark handleObjectEnd(fpFlowAfterObject)
anchor = yAnchorNone
level = ancestry.pop()
state = fpFlowAfterObject
of '"': of '"':
content = "" content = ""
lexer.doublyQuotedScalar(content) lexer.doublyQuotedScalar(content)
if tag == yTagQuestionmark: if tag == yTagQuestionmark:
tag = yTagExclamationmark tag = yTagExclamationmark
yield scalarEvent(content, tag, anchor) yield scalarEvent(content, tag, anchor)
tag = yTagQuestionmark handleObjectEnd(fpFlowAfterObject)
anchor = yAnchorNone
level = ancestry.pop()
state = fpFlowAfterObject
of '!': of '!':
handleTagHandle() handleTagHandle()
of '&': of '&':
handleAnchor() handleAnchor()
of '*': of '*':
handleAlias() handleAlias()
level = ancestry.pop()
yield cachedScalar
state = fpFlowAfterObject state = fpFlowAfterObject
else: else:
handleFlowPlainScalar() handleFlowPlainScalar()
of fpFlowAfterObject: of fpFlowAfterObject:
debug("state: flowAfterObject")
lexer.skipWhitespaceAndNewlines() lexer.skipWhitespaceAndNewlines()
case lexer.buf[lexer.bufpos] case lexer.buf[lexer.bufpos]
of ']': of ']':
case level.kind case level.kind
of fplSequence: of fplSequence:
yield endSeqEvent() discard
of fplMapKey, fplMapValue: of fplMapKey, fplMapValue:
raiseError("Unexpected token (expected '}')", lexer.bufpos) raiseError("Unexpected token (expected '}')", lexer.bufpos)
of fplScalar, fplUnknown: of fplScalar, fplUnknown:
assert(false) assert(false)
lexer.bufpos.inc()
leaveFlowLevel() leaveFlowLevel()
of '}': of '}':
case level.kind case level.kind
of [fplMapKey, fplMapValue]:
discard
of fplSequence: of fplSequence:
raiseError("Unexpected token (expected ']')", lexer.bufpos) raiseError("Unexpected token (expected ']')", lexer.bufpos)
of fplMapKey:
yield scalarEvent("", yTagQuestionmark, yAnchorNone)
of fplMapValue:
discard
of fplUnknown, fplScalar: of fplUnknown, fplScalar:
assert(false) assert(false)
yield endMapEvent() lexer.bufpos.inc()
leaveFlowLevel() leaveFlowLevel()
of ',': of ',':
case level.kind case level.kind
of fplSequence: of fplSequence:
discard discard
of fplMapKey:
yield scalarEvent("", yTagQuestionmark, yAnchorNone)
of fplMapValue: of fplMapValue:
yield scalarEvent("", yTagQuestionmark, yAnchorNone)
level.kind = fplMapKey level.kind = fplMapKey
of fplMapKey:
discard
of fplUnknown, fplScalar: of fplUnknown, fplScalar:
assert(false) assert(false)
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown) level = FastParseLevel(kind: fplUnknown)
state = fpFlow state = fpFlow
lexer.bufpos.inc()
of ':': of ':':
case level.kind case level.kind
of fplSequence, fplMapValue: of fplSequence, fplMapKey:
raiseError("Unexpected token (expected ',')", lexer.bufpos) raiseError("Unexpected token (expected ',')", lexer.bufpos)
of fplMapKey: of fplMapValue:
level.kind = fplMapValue level.kind = fplMapValue
of fplUnknown, fplScalar: of fplUnknown, fplScalar:
assert(false) assert(false)
ancestry.add(level) ancestry.add(level)
level = FastParseLevel(kind: fplUnknown) level = FastParseLevel(kind: fplUnknown)
state = fpFlow state = fpFlow
lexer.bufpos.inc()
of '#': of '#':
lexer.lineEnding() lexer.lineEnding()
handleLineEnd(true)
of EndOfFile: of EndOfFile:
raiseError("Unclosed flow content", lexer.bufpos) raiseError("Unclosed flow content", lexer.bufpos)
else: else:
raiseError("Unexpected content (expected flow indicator)", raiseError("Unexpected content (expected flow indicator)",
lexer.bufpos) lexer.bufpos)
lexer.bufpos.inc()