Implemented block scalars

This commit is contained in:
Felix Krause 2015-12-17 21:44:41 +01:00
parent 3ca3081e92
commit edec7ece37
3 changed files with 141 additions and 11 deletions

View File

@ -25,8 +25,7 @@ type
yamlOpeningBracket, yamlClosingBrace, yamlClosingBracket, yamlPipe,
yamlGreater,
# block scalar header
yamlLiteralScalar, yamlFoldedScalar,
yamlBlockIndentationIndicator, yamlBlockChompingIndicator,
yamlBlockIndentationIndicator, yamlPlus,
# scalar content
yamlScalar, yamlScalarPart,
# tags
@ -802,9 +801,10 @@ iterator tokens*(my: var YamlLexer): YamlLexerToken {.closure.} =
of '0' .. '9':
my.content = "" & c
yieldToken(yamlBlockIndentationIndicator)
of '+', '-':
my.content = "" & c
yieldToken(yamlBlockChompingIndicator)
of '+':
yieldToken(yamlPlus)
of '-':
yieldToken(yamlDash)
of '\r', '\x0A', EndOfFile:
blockScalarIndentation = lastIndentationLength
state = ylLineEnd

View File

@ -29,18 +29,24 @@ type
YamlParserState = enum
ylInitial, ylSkipDirective, ylBlockLineStart, ylBlockAfterTag,
ylBlockAfterAnchor, ylBlockAfterScalar, ylBlockAfterColon,
ylBlockMultilineScalar, ylBlockLineEnd, ylFlow, ylFlowAfterObject,
ylExpectingDocumentEnd
ylBlockMultilineScalar, ylBlockLineEnd, ylBlockScalarHeader,
ylBlockScalar, ylFlow, ylFlowAfterObject, ylExpectingDocumentEnd
DocumentLevelMode = enum
mBlockSequenceItem, mFlowSequenceItem, mExplicitBlockMapKey,
mExplicitBlockMapValue, mImplicitBlockMapKey, mImplicitBlockMapValue,
mFlowMapKey, mFlowMapValue, mPlainScalar, mScalar, mUnknown
mFlowMapKey, mFlowMapValue, mScalar, mUnknown
DocumentLevel = object
mode: DocumentLevelMode
indicatorColumn: int
indentationColumn: int
LineStrippingMode = enum
lsStrip, lsClip, lsKeep
BlockScalarStyle = enum
bsLiteral, bsFolded
proc `==`*(left: YamlParserEvent, right: YamlParserEvent): bool =
if left.kind != right.kind:
@ -156,6 +162,12 @@ iterator events*(input: Stream): YamlParserEvent {.closure.} =
level = DocumentLevel(mode: mUnknown, indicatorColumn: -1,
indentationColumn: -1)
# block scalar state
lineStrip: LineStrippingMode
blockScalar: BlockScalarStyle
blockScalarIndentation: int
blockScalarTrailing: string = nil
# cached values
tag: string = nil
anchor: string = nil
@ -239,6 +251,13 @@ iterator events*(input: Stream): YamlParserEvent {.closure.} =
of yamlColon:
handleBlockIndicator(mExplicitBlockMapKey,
mExplicitBlockMapValue, yamlError)
of yamlPipe, yamlGreater:
blockScalar = if token == yamlPipe: bsLiteral else: bsFolded
blockScalarIndentation = -1
lineStrip = lsClip
state = ylBlockScalarHeader
scalarCache = ""
level.mode = mScalar
of yamlTagHandle:
let handle = lex.content
if tagShorthands.hasKey(handle):
@ -431,6 +450,13 @@ iterator events*(input: Stream): YamlParserEvent {.closure.} =
of yamlOpeningBracket, yamlOpeningBrace:
state = ylFlow
continue
of yamlPipe, yamlGreater:
blockScalar = if token == yamlPipe: bsLiteral else: bsFolded
blockScalarIndentation = -1
lineStrip = lsClip
state = ylBlockScalarHeader
scalarCache = ""
level.mode = mScalar
else:
yieldError("Unexpected token (expected scalar or line end): " &
$token)
@ -445,6 +471,84 @@ iterator events*(input: Stream): YamlParserEvent {.closure.} =
break
else:
yieldError("Unexpected token (expected line end):" & $token)
of ylBlockScalarHeader:
case token
of yamlPlus:
if lineStrip != lsClip:
yieldError("Multiple chomping indicators!")
else:
lineStrip = lsKeep
of yamlDash:
if lineStrip != lsClip:
yieldError("Multiple chomping indicators!")
else:
lineStrip = lsStrip
of yamlBlockIndentationIndicator:
if blockScalarIndentation != -1:
yieldError("Multiple indentation indicators!")
else:
blockScalarIndentation = parseInt(lex.content)
of yamlLineStart:
blockScalarTrailing = ""
state = ylBlockScalar
else:
yieldError("Unexpected token: " & $token)
of ylBlockScalar:
case token
of yamlLineStart:
if level.indentationColumn == -1:
discard
else:
case blockScalar
of bsLiteral:
blockScalarTrailing &= "\x0A"
of bsFolded:
case blockScalarTrailing.len
of 0:
blockScalarTrailing = " "
of 1:
blockScalarTrailing = "\x0A"
else:
discard
if lex.content.len > level.indentationColumn:
if blockScalar == bsFolded:
if blockScalarTrailing == " ":
blockScalarTrailing = "\x0A"
scalarCache &= blockScalarTrailing &
lex.content[level.indentationColumn..^1]
blockScalarTrailing = ""
of yamlScalarPart:
if ancestry.high > 0:
if ancestry[ancestry.high].indicatorColumn >= lex.column or
ancestry[ancestry.high].indicatorColumn == -1 and
ancestry[ancestry.high].indentationColumn >= lex.column:
# todo: trailing chomping?
closeLevel(level)
state = ylBlockLineStart
continue
if level.indentationColumn == -1:
level.indentationColumn = lex.column
else:
scalarCache &= blockScalarTrailing
blockScalarTrailing = ""
scalarCache &= lex.content
else:
case lineStrip
of lsStrip:
discard
of lsClip:
scalarCache &= "\x0A"
of lsKeep:
scalarCache &= blockScalarTrailing
closeLevel(level)
if ancestry.len == 0:
state = ylExpectingDocumentEnd
else:
level = ancestry.pop()
state = ylBlockLineStart
continue
of ylFlow:
case token
of yamlLineStart:

View File

@ -57,8 +57,20 @@ proc printDifference(expected, actual: YamlParserEvent) =
echo "[scalar] expected anchor " & expected.scalarAnchor &
", got " & actual.scalarAnchor
elif expected.scalarContent != actual.scalarContent:
echo "[scalar] expected content \"" & expected.scalarContent &
"\", got \"" & actual.scalarContent & "\""
let msg = "[scalar] expected content \"" &
expected.scalarContent & "\", got \"" &
actual.scalarContent & "\" "
for i in 0..expected.scalarContent.high:
if i >= actual.scalarContent.high:
echo msg, "(expected more chars, first char missing: ",
cast[int](expected.scalarContent[i]), ")"
break
elif expected.scalarContent[i] != actual.scalarContent[i]:
echo msg, "(first different char at pos ", i,
": expected ",
cast[int](expected.scalarContent[i]), ", got ",
cast[int](actual.scalarContent[i]), ")"
break
else:
echo "[scalar] Unknown difference"
else:
@ -132,4 +144,18 @@ suite "Parsing":
ensure("a\nb \n c\nd", startDoc(), scalar("a b c d"), endDoc())
test "Parsing: Multiline scalar (in map)":
ensure("a: b\n c\nd:\n e\n f", startDoc(), startMap(), scalar("a"),
scalar("b c"), scalar("d"), scalar("e f"), endMap(), endDoc())
scalar("b c"), scalar("d"), scalar("e f"), endMap(), endDoc())
test "Parsing: Block scalar (literal)":
ensure("a: |\x0A ab\x0A \x0A cd\x0A ef\x0A \x0A", startDoc(),
startMap(), scalar("a"), scalar("ab\x0A\x0Acd\x0Aef\x0A"),
endMap(), endDoc())
test "Parsing: Block scalar (folded)":
ensure("a: >\x0A ab\x0A cd\x0A \x0Aef\x0A\x0A\x0Agh\x0A", startDoc(),
startMap(), scalar("a"), scalar("ab cd\x0Aef\x0Agh\x0A"),
endMap(), endDoc())
test "Parsing: Block scalar (keep)":
ensure("a: |+\x0A ab\x0A \x0A \x0A", startDoc(), startMap(),
scalar("a"), scalar("ab\x0A\x0A \x0A"), endMap(), endDoc())
test "Parsing: Block scalar (strip)":
ensure("a: |-\x0A ab\x0A \x0A \x0A", startDoc(), startMap(),
scalar("a"), scalar("ab"), endMap(), endDoc())