status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

started making serializationTests green again

This commit is contained in:
Felix Krause 2020-11-06 21:39:50 +01:00
parent 48d601d959
commit 6238104622
8 changed files with 397 additions and 384 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
test/tlex.nim
View File

@ -210,6 +210,9 @@ suite "Lexer":
assertEquals("foo:\n bar\n baz\n---\nderp", i(0), pl("foo"), mv(), i(2), assertEquals("foo:\n bar\n baz\n---\nderp", i(0), pl("foo"), mv(), i(2),
pl("bar baz"), dirE(), i(0), pl("derp"), e()) pl("bar baz"), dirE(), i(0), pl("derp"), e())
test "Sequence with compact maps":
assertEquals("""- a: drzw\n- b""", i(0), ss(), pl("a"), mv(), pl("drzw"), i(0), ss(), pl("b"), e())
test "Empty lines": test "Empty lines":
assertEquals("""block: foo assertEquals("""block: foo

22
test/tserialization.nim
View File

@ -311,17 +311,17 @@ suite "Serialization":
input[(a: 13'i32, b: 47'i32)] = "dreizehnsiebenundvierzig" input[(a: 13'i32, b: 47'i32)] = "dreizehnsiebenundvierzig"
var output = dump(input, tsRootOnly, asTidy, blockOnly) var output = dump(input, tsRootOnly, asTidy, blockOnly)
assertStringEqual(yamlDirs & assertStringEqual(yamlDirs &
"""!n!tables:OrderedTable(tag:nimyaml.org;2016:tuple(tag:nimyaml.org;2016:system:int32;tag:nimyaml.org;2016:system:int32);tag:yaml.org;2002:str) "!n!tables:OrderedTable(tag:nimyaml.org;2016:tuple(tag:nimyaml.org;2016:system:int32;tag:nimyaml.org;2016:system:int32);tag:yaml.org;2002:str) \n" &
- "- \n" &
? " ? \n" &
a: 23 " a: 23\n" &
b: 42 " b: 42\n" &
: dreiundzwanzigzweiundvierzig " : dreiundzwanzigzweiundvierzig\n" &
- "- \n" &
? " ? \n" &
a: 13 " a: 13\n" &
b: 47 " b: 47\n" &
: dreizehnsiebenundvierzig""", output) " : dreizehnsiebenundvierzig", output)
test "Load Sequences in Sequence": test "Load Sequences in Sequence":
let input = " - [1, 2, 3]\n - [4, 5]\n - [6]" let input = " - [1, 2, 3]\n - [4, 5]\n - [6]"

7
yaml/data.nim
View File

@ -68,6 +68,7 @@ type
of yamlStartDoc: of yamlStartDoc:
explicitDirectivesEnd*: bool explicitDirectivesEnd*: bool
version*: string version*: string
handles*: seq[tuple[handle, uriPrefix: string]]
of yamlEndDoc: of yamlEndDoc:
explicitDocumentEnd*: bool explicitDocumentEnd*: bool
of yamlEndMap, yamlEndSeq: discard of yamlEndMap, yamlEndSeq: discard
@ -162,11 +163,13 @@ proc startStreamEvent*(): Event =
proc endStreamEvent*(): Event = proc endStreamEvent*(): Event =
return Event(startPos: defaultMark, endPos: defaultMark, kind: yamlEndStream) return Event(startPos: defaultMark, endPos: defaultMark, kind: yamlEndStream)
proc startDocEvent*(explicit: bool = false, version: string = "", startPos, endPos: Mark = defaultMark): Event proc startDocEvent*(explicit: bool = false, version: string = "",
handles: seq[tuple[handle, uriPrefix: string]] = @[],
startPos, endPos: Mark = defaultMark): Event
{.inline, raises: [].} = {.inline, raises: [].} =
## creates a new event that marks the start of a YAML document ## creates a new event that marks the start of a YAML document
result = Event(startPos: startPos, endPos: endPos, result = Event(startPos: startPos, endPos: endPos,
kind: yamlStartDoc, version: version, kind: yamlStartDoc, version: version, handles: handles,
explicitDirectivesEnd: explicit) explicitDirectivesEnd: explicit)
proc endDocEvent*(explicit: bool = false, startPos, endPos: Mark = defaultMark): Event proc endDocEvent*(explicit: bool = false, startPos, endPos: Mark = defaultMark): Event

90
yaml/parser.nim
View File

@ -35,6 +35,7 @@ type
Context = ref object of YamlStream Context = ref object of YamlStream
tagLib: TagLibrary tagLib: TagLibrary
handles: seq[tuple[handle, uriPrefix: string]]
issueWarnings: bool issueWarnings: bool
lex: Lexer lex: Lexer
levels: seq[Level] levels: seq[Level]
@ -121,7 +122,7 @@ proc afterFlowMapValue(c: Context, e: var Event): bool
proc afterFlowSeqSepProps(c: Context, e: var Event): bool proc afterFlowSeqSepProps(c: Context, e: var Event): bool
proc afterFlowSeqItem(c: Context, e: var Event): bool proc afterFlowSeqItem(c: Context, e: var Event): bool
proc afterPairValue(c: Context, e: var Event): bool proc afterPairValue(c: Context, e: var Event): bool
proc emitCollectionKey(c: Context, e: var Event): bool proc emitCached(c: Context, e: var Event): bool
{.pop.} {.pop.}
template debug(message: string) {.dirty.} = template debug(message: string) {.dirty.} =
@ -153,6 +154,12 @@ template popLevel(c: Context) =
debug("parser: pop") debug("parser: pop")
discard c.levels.pop() discard c.levels.pop()
proc resolveHandle(c: Context, handle: string): string {.raises: [].} =
for item in c.handles:
if item.handle == handle:
return item.uriPrefix
return ""
proc init[T](c: Context, p: YamlParser, source: T) {.inline.} = proc init[T](c: Context, p: YamlParser, source: T) {.inline.} =
c.pushLevel(atStreamStart, -2) c.pushLevel(atStreamStart, -2)
c.nextImpl = proc(s: YamlStream, e: var Event): bool = c.nextImpl = proc(s: YamlStream, e: var Event): bool =
@ -207,7 +214,7 @@ proc generateError(c: Context, message: string):
proc parseTag(c: Context): TagId = proc parseTag(c: Context): TagId =
let handle = c.lex.fullLexeme() let handle = c.lex.fullLexeme()
var uri = c.tagLib.resolve(handle) var uri = c.resolveHandle(handle)
if uri == "": if uri == "":
raise c.generateError("unknown handle: " & escape(handle)) raise c.generateError("unknown handle: " & escape(handle))
c.lex.next() c.lex.next()
@ -251,7 +258,7 @@ proc atStreamStart(c: Context, e: var Event): bool =
c.pushLevel(beforeDoc, -1) c.pushLevel(beforeDoc, -1)
e = Event(startPos: c.lex.curStartPos, endPos: c.lex.curStartPos, kind: yamlStartStream) e = Event(startPos: c.lex.curStartPos, endPos: c.lex.curStartPos, kind: yamlStartStream)
c.lex.next() c.lex.next()
c.tagLib.resetPrefixes() resetHandles(c.handles)
return true return true
proc atStreamEnd(c: Context, e : var Event): bool = proc atStreamEnd(c: Context, e : var Event): bool =
@ -269,7 +276,7 @@ proc beforeDoc(c: Context, e: var Event): bool =
raise c.generateError("Missing `---` after directives") raise c.generateError("Missing `---` after directives")
c.lex.next() c.lex.next()
of DirectivesEnd: of DirectivesEnd:
e = startDocEvent(true, version, c.lex.curStartPos, c.lex.curEndPos) e = startDocEvent(true, version, c.handles, c.lex.curStartPos, c.lex.curEndPos)
c.lex.next() c.lex.next()
c.transition(beforeDocEnd) c.transition(beforeDocEnd)
c.pushLevel(afterDirectivesEnd, -1) c.pushLevel(afterDirectivesEnd, -1)
@ -278,7 +285,7 @@ proc beforeDoc(c: Context, e: var Event): bool =
c.popLevel() c.popLevel()
return false return false
of Indentation: of Indentation:
e = startDocEvent(false, version, c.lex.curStartPos, c.lex.curEndPos) e = startDocEvent(false, version, c.handles, c.lex.curStartPos, c.lex.curEndPos)
c.transition(beforeDocEnd) c.transition(beforeDocEnd)
c.pushLevel(beforeImplicitRoot, -1) c.pushLevel(beforeImplicitRoot, -1)
return true return true
@ -302,7 +309,7 @@ proc beforeDoc(c: Context, e: var Event): bool =
c.lex.next() c.lex.next()
if c.lex.cur != Token.Suffix: if c.lex.cur != Token.Suffix:
raise c.generateError("Invalid token (expected tag URI): " & $c.lex.cur) raise c.generateError("Invalid token (expected tag URI): " & $c.lex.cur)
discard c.tagLib.registerHandle(c.lex.evaluated, tagHandle) discard registerHandle(c.handles, tagHandle, c.lex.evaluated)
c.lex.next() c.lex.next()
of UnknownDirective: of UnknownDirective:
seenDirectives = true seenDirectives = true
@ -407,9 +414,10 @@ proc atBlockIndentation(c: Context, e: var Event): bool =
raise c.generateError("Implicit mapping key may not be multiline") raise c.generateError("Implicit mapping key may not be multiline")
let props = e.scalarProperties let props = e.scalarProperties
e.scalarProperties = autoScalarTag(defaultProperties, scalarToken) e.scalarProperties = autoScalarTag(defaultProperties, scalarToken)
c.peek = move(e) c.keyCache.add(move(e))
e = startMapEvent(csBlock, props, c.headerStart, headerEnd) e = startMapEvent(csBlock, props, c.headerStart, headerEnd)
c.transition(afterImplicitKey) c.transition(afterImplicitKey)
c.pushLevel(emitCached)
else: else:
e.scalarProperties = autoScalarTag(e.scalarProperties, scalarToken) e.scalarProperties = autoScalarTag(e.scalarProperties, scalarToken)
c.popLevel() c.popLevel()
@ -420,10 +428,11 @@ proc atBlockIndentation(c: Context, e: var Event): bool =
let headerEnd = c.lex.curStartPos let headerEnd = c.lex.curStartPos
c.lex.next() c.lex.next()
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
c.peek = move(e) c.keyCache.add(move(e))
e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.transition(afterImplicitKey) c.transition(afterImplicitKey)
c.pushLevel(emitCached)
elif not isEmpty(c.headerProps): elif not isEmpty(c.headerProps):
raise c.generateError("Alias may not have properties") raise c.generateError("Alias may not have properties")
else: else:
@ -437,11 +446,12 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
c.updateIndentation(c.lex.recentIndentation()) c.updateIndentation(c.lex.recentIndentation())
case c.lex.cur case c.lex.cur
of MapValueInd: of MapValueInd:
c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos) c.keyCache.add(scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curEndPos))
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
e = startMapEvent(csBlock, c.headerProps, c.lex.curStartPos, c.lex.curEndPos) e = startMapEvent(csBlock, c.headerProps, c.lex.curStartPos, c.lex.curEndPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.transition(afterImplicitKey) c.transition(afterImplicitKey)
c.pushLevel(emitCached)
return true return true
of Plain, SingleQuoted, DoubleQuoted: of Plain, SingleQuoted, DoubleQuoted:
e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur), e = scalarEvent(c.lex.evaluated, autoScalarTag(c.inlineProps, c.lex.cur),
@ -452,10 +462,11 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
if c.lex.lastScalarWasMultiline(): if c.lex.lastScalarWasMultiline():
raise c.generateError("Implicit mapping key may not be multiline") raise c.generateError("Implicit mapping key may not be multiline")
c.peek = move(e) c.keyCache.add(move(e))
e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd) e = startMapEvent(csBlock, c.headerProps, c.headerStart, headerEnd)
c.headerProps = defaultProperties c.headerProps = defaultProperties
c.transition(afterImplicitKey) c.transition(afterImplicitKey)
c.pushLevel(emitCached)
else: else:
c.mergeProps(c.headerProps, e.scalarProperties) c.mergeProps(c.headerProps, e.scalarProperties)
c.popLevel() c.popLevel()
@ -463,23 +474,23 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
of MapStart, SeqStart: of MapStart, SeqStart:
let let
startPos = c.lex.curStartPos startPos = c.lex.curStartPos
indent = c.levels[^1].indentation indent = c.lex.currentIndentation()
levelDepth = c.levels.len
c.transition(beforeFlowItemProps) c.transition(beforeFlowItemProps)
c.caching = true c.caching = true
while c.lex.flowDepth > 0: while c.levels.len >= levelDepth:
c.keyCache.add(c.next()) c.keyCache.add(c.next())
c.keyCache.add(c.next())
c.caching = false c.caching = false
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
c.pushLevel(afterImplicitKey, indent) c.pushLevel(afterImplicitKey, indent)
c.pushLevel(emitCollectionKey) c.pushLevel(emitCached)
if c.lex.curStartPos.line != startPos.line: if c.lex.curStartPos.line != startPos.line:
raise c.generateError("Implicit mapping key may not be multiline") raise c.generateError("Implicit mapping key may not be multiline")
e = startMapEvent(csBlock, c.headerProps, c.headerStart, startPos) e = startMapEvent(csBlock, c.headerProps, c.headerStart, startPos)
c.headerProps = defaultProperties c.headerProps = defaultProperties
return true return true
else: else:
c.pushLevel(emitCollectionKey) c.pushLevel(emitCached)
return false return false
of Literal, Folded: of Literal, Folded:
c.mergeProps(c.inlineProps, c.headerProps) c.mergeProps(c.inlineProps, c.headerProps)
@ -491,6 +502,7 @@ proc atBlockIndentationProps(c: Context, e: var Event): bool =
return true return true
of Indentation: of Indentation:
c.lex.next() c.lex.next()
c.transition(atBlockIndentation)
return false return false
else: else:
raise c.generateError("Unexpected token (expected block content): " & $c.lex.cur) raise c.generateError("Unexpected token (expected block content): " & $c.lex.cur)
@ -567,19 +579,21 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
c.popLevel() c.popLevel()
return true return true
of MapValueInd: of MapValueInd:
c.peek = scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos) c.keyCache.add(scalarEvent("", c.inlineProps, ssPlain, c.inlineStart, c.lex.curStartPos))
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
e = startMapEvent(csBlock, defaultProperties, c.lex.curStartPos, c.lex.curStartPos) e = startMapEvent(csBlock, defaultProperties, c.lex.curStartPos, c.lex.curStartPos)
c.transition(afterImplicitKey) c.transition(afterImplicitKey)
c.pushLevel(emitCached)
return true return true
of Alias: of Alias:
e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos) e = aliasEvent(c.lex.shortLexeme().Anchor, c.inlineStart, c.lex.curEndPos)
let headerEnd = c.lex.curStartPos let headerEnd = c.lex.curStartPos
c.lex.next() c.lex.next()
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
c.peek = move(e) c.keyCache.add(move(e))
e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd) e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd)
c.transition(afterImplicitKey) c.transition(afterImplicitKey)
c.pushLevel(emitCached)
else: else:
c.popLevel() c.popLevel()
return true return true
@ -593,24 +607,16 @@ proc afterCompactParentProps(c: Context, e: var Event): bool =
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
if c.lex.lastScalarWasMultiline(): if c.lex.lastScalarWasMultiline():
raise c.generateError("Implicit mapping key may not be multiline") raise c.generateError("Implicit mapping key may not be multiline")
c.peek = move(e) c.keyCache.add(move(e))
e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd) e = startMapEvent(csBlock, defaultProperties, headerEnd, headerEnd)
c.transition(afterImplicitKey) c.transition(afterImplicitKey)
c.pushLevel(emitCached)
else: else:
c.popLevel() c.popLevel()
return true return true
of MapStart: of MapStart, SeqStart:
e = startMapEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos) c.transition(atBlockIndentationProps)
c.inlineProps = defaultProperties return false
c.transition(afterFlowMapSep)
c.lex.next()
return true
of SeqStart:
e = startSeqEvent(csFlow, c.inlineProps, c.inlineStart, c.lex.curEndPos)
c.inlineProps = defaultProperties
c.transition(afterFlowSeqSep)
c.lex.next()
return true
else: else:
raise c.generateError("Unexpected token (expected newline or flow item start: " & $c.lex.cur) raise c.generateError("Unexpected token (expected newline or flow item start: " & $c.lex.cur)
@ -660,14 +666,14 @@ proc beforeDocEnd(c: Context, e: var Event): bool =
e = endDocEvent(true, c.lex.curStartPos, c.lex.curEndPos) e = endDocEvent(true, c.lex.curStartPos, c.lex.curEndPos)
c.transition(beforeDoc) c.transition(beforeDoc)
c.lex.next() c.lex.next()
c.tagLib.resetPrefixes() resetHandles(c.handles)
of StreamEnd: of StreamEnd:
e = endDocEvent(false, c.lex.curStartPos, c.lex.curEndPos) e = endDocEvent(false, c.lex.curStartPos, c.lex.curEndPos)
c.popLevel() c.popLevel()
of DirectivesEnd: of DirectivesEnd:
e = endDocEvent(false, c.lex.curStartPos, c.lex.curStartPos) e = endDocEvent(false, c.lex.curStartPos, c.lex.curStartPos)
c.transition(beforeDoc) c.transition(beforeDoc)
c.tagLib.resetPrefixes() resetHandles(c.handles)
else: else:
raise c.generateError("Unexpected token (expected document end): " & $c.lex.cur) raise c.generateError("Unexpected token (expected document end): " & $c.lex.cur)
return true return true
@ -693,6 +699,7 @@ proc inBlockSeq(c: Context, e: var Event): bool =
proc beforeBlockMapKey(c: Context, e: var Event): bool = proc beforeBlockMapKey(c: Context, e: var Event): bool =
if c.blockIndentation > c.levels[^1].indentation: if c.blockIndentation > c.levels[^1].indentation:
raise c.generateError("Invalid indentation: got " & $c.blockIndentation & ", expected " & $c.levels[^1].indentation) raise c.generateError("Invalid indentation: got " & $c.blockIndentation & ", expected " & $c.levels[^1].indentation)
c.inlineStart = c.lex.curStartPos
case c.lex.cur case c.lex.cur
of MapKeyInd: of MapKeyInd:
c.transition(beforeBlockMapValue) c.transition(beforeBlockMapValue)
@ -959,29 +966,32 @@ proc afterFlowSeqSepProps(c: Context, e: var Event): bool =
c.inlineProps = defaultProperties c.inlineProps = defaultProperties
c.lex.next() c.lex.next()
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
c.peek = move(e)
e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curStartPos)
c.pushLevel(afterImplicitPairStart) c.pushLevel(afterImplicitPairStart)
if c.caching:
c.keyCache.add(startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curStartPos))
else:
c.keyCache.add(move(e))
e = startMapEvent(csFlow, defaultProperties, c.lex.curStartPos, c.lex.curStartPos)
c.pushLevel(emitCached)
return true return true
of MapStart, SeqStart: of MapStart, SeqStart:
let let
startPos = c.lex.curStartPos startPos = c.lex.curStartPos
indent = c.levels[^1].indentation indent = c.levels[^1].indentation
cacheStart = c.keyCache.len cacheStart = c.keyCache.len
targetFlowDepth = c.lex.flowDepth - 1 levelDepth = c.levels.len
alreadyCaching = c.caching alreadyCaching = c.caching
c.pushLevel(beforeFlowItemProps) c.pushLevel(beforeFlowItemProps)
c.caching = true c.caching = true
while c.lex.flowDepth > targetFlowDepth: while c.levels.len > levelDepth:
c.keyCache.add(c.next()) c.keyCache.add(c.next())
c.keyCache.add(c.next())
c.caching = alreadyCaching c.caching = alreadyCaching
if c.lex.cur == Token.MapValueInd: if c.lex.cur == Token.MapValueInd:
c.pushLevel(afterImplicitPairStart, indent) c.pushLevel(afterImplicitPairStart, indent)
if c.lex.curStartPos.line != startPos.line: if c.lex.curStartPos.line != startPos.line:
raise c.generateError("Implicit mapping key may not be multiline") raise c.generateError("Implicit mapping key may not be multiline")
if not alreadyCaching: if not alreadyCaching:
c.pushLevel(emitCollectionKey) c.pushLevel(emitCached)
e = startMapEvent(csPair, defaultProperties, startPos, startPos) e = startMapEvent(csPair, defaultProperties, startPos, startPos)
return true return true
else: else:
@ -991,7 +1001,7 @@ proc afterFlowSeqSepProps(c: Context, e: var Event): bool =
return false return false
else: else:
if not alreadyCaching: if not alreadyCaching:
c.pushLevel(emitCollectionKey) c.pushLevel(emitCached)
return false return false
else: else:
c.pushLevel(beforeFlowItem) c.pushLevel(beforeFlowItem)
@ -1025,7 +1035,7 @@ proc afterPairValue(c: Context, e: var Event): bool =
c.popLevel() c.popLevel()
return true return true
proc emitCollectionKey(c: Context, e: var Event): bool = proc emitCached(c: Context, e: var Event): bool =
debug("emitCollection key: pos = " & $c.keyCachePos & ", len = " & $c.keyCache.len) debug("emitCollection key: pos = " & $c.keyCachePos & ", len = " & $c.keyCache.len)
yAssert(c.keyCachePos < c.keyCache.len) yAssert(c.keyCachePos < c.keyCache.len)
e = move(c.keyCache[c.keyCachePos]) e = move(c.keyCache[c.keyCachePos])

557
yaml/presenter.nim
View File

@ -107,7 +107,12 @@ type
ScalarStyle = enum ScalarStyle = enum
sLiteral, sFolded, sPlain, sDoubleQuoted sLiteral, sFolded, sPlain, sDoubleQuoted
PresenterTarget = Stream | ptr[string] Context = object
target: Stream
tagLib: TagLibrary
options: PresentationOptions
handles: seq[tuple[handle, uriPrefix: string]]
levels: seq[DumperState]
const const
defaultPresentationOptions* = defaultPresentationOptions* =
@ -124,6 +129,24 @@ proc defineOptions*(style: PresentationStyle = psDefault,
PresentationOptions(style: style, indentationStep: indentationStep, PresentationOptions(style: style, indentationStep: indentationStep,
newlines: newlines, outputVersion: outputVersion) newlines: newlines, outputVersion: outputVersion)
proc state(c: Context): DumperState = c.levels[^1]
proc `state=`(c: var Context, v: DumperState) =
c.levels[^1] = v
proc searchHandle(c: Context, tag: string):
tuple[handle: string, len: int] {.raises: [].} =
## search in the registered tag handles for one whose prefix matches the start
## of the given tag. If multiple registered handles match, the one with the
## longest prefix is returned. If no registered handle matches, ("", 0) is
## returned.
result.len = 0
for item in c.handles:
if item.uriPrefix.len > result.len:
if tag.startsWith(item.uriPrefix):
result.len = item.uriPrefix.len
result.handle = item.handle
proc inspect(scalar: string, indentation: int, proc inspect(scalar: string, indentation: int,
words, lines: var seq[tuple[start, finish: int]]): words, lines: var seq[tuple[start, finish: int]]):
ScalarStyle {.raises: [].} = ScalarStyle {.raises: [].} =
@ -204,111 +227,114 @@ template append(target: Stream, val: string | char) =
template append(target: ptr[string], val: string | char) = template append(target: ptr[string], val: string | char) =
target[].add(val) target[].add(val)
proc writeDoubleQuoted(scalar: string, s: PresenterTarget, indentation: int, proc writeDoubleQuoted(c: Context, scalar: string, indentation: int,
newline: string) newline: string)
{.raises: [YamlPresenterOutputError].} = {.raises: [YamlPresenterOutputError].} =
var curPos = indentation var curPos = indentation
let t = c.target
try: try:
s.append('"') t.append('"')
curPos.inc() curPos.inc()
for c in scalar: for c in scalar:
if curPos == 79: if curPos == 79:
s.append('\\') t.append('\\')
s.append(newline) t.append(newline)
s.append(repeat(' ', indentation)) t.append(repeat(' ', indentation))
curPos = indentation curPos = indentation
if c == ' ': if c == ' ':
s.append('\\') t.append('\\')
curPos.inc() curPos.inc()
case c case c
of '"': of '"':
s.append("\\\"") t.append("\\\"")
curPos.inc(2) curPos.inc(2)
of '\l': of '\l':
s.append("\\n") t.append("\\n")
curPos.inc(2) curPos.inc(2)
of '\t': of '\t':
s.append("\\t") t.append("\\t")
curPos.inc(2) curPos.inc(2)
of '\\': of '\\':
s.append("\\\\") t.append("\\\\")
curPos.inc(2) curPos.inc(2)
else: else:
if ord(c) < 32: if ord(c) < 32:
s.append("\\x" & toHex(ord(c), 2)) t.append("\\x" & toHex(ord(c), 2))
curPos.inc(4) curPos.inc(4)
else: else:
s.append(c) t.append(c)
curPos.inc() curPos.inc()
s.append('"') t.append('"')
except: except:
var e = newException(YamlPresenterOutputError, var e = newException(YamlPresenterOutputError,
"Error while writing to output stream") "Error while writing to output stream")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
proc writeDoubleQuotedJson(scalar: string, s: PresenterTarget) proc writeDoubleQuotedJson(c: Context, scalar: string)
{.raises: [YamlPresenterOutputError].} = {.raises: [YamlPresenterOutputError].} =
let t = c.target
try: try:
s.append('"') t.append('"')
for c in scalar: for c in scalar:
case c case c
of '"': s.append("\\\"") of '"': t.append("\\\"")
of '\\': s.append("\\\\") of '\\': t.append("\\\\")
of '\l': s.append("\\n") of '\l': t.append("\\n")
of '\t': s.append("\\t") of '\t': t.append("\\t")
of '\f': s.append("\\f") of '\f': t.append("\\f")
of '\b': s.append("\\b") of '\b': t.append("\\b")
else: else:
if ord(c) < 32: s.append("\\u" & toHex(ord(c), 4)) else: s.append(c) if ord(c) < 32: t.append("\\u" & toHex(ord(c), 4)) else: t.append(c)
s.append('"') t.append('"')
except: except:
var e = newException(YamlPresenterOutputError, var e = newException(YamlPresenterOutputError,
"Error while writing to output stream") "Error while writing to output stream")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
proc writeLiteral(scalar: string, indentation, indentStep: int, proc writeLiteral(c: Context, scalar: string, indentation, indentStep: int,
s: PresenterTarget, lines: seq[tuple[start, finish: int]], lines: seq[tuple[start, finish: int]], newline: string)
newline: string)
{.raises: [YamlPresenterOutputError].} = {.raises: [YamlPresenterOutputError].} =
let t = c.target
try: try:
s.append('|') t.append('|')
if scalar[^1] != '\l': s.append('-') if scalar[^1] != '\l': t.append('-')
if scalar[0] in [' ', '\t']: s.append($indentStep) if scalar[0] in [' ', '\t']: t.append($indentStep)
for line in lines: for line in lines:
s.append(newline) t.append(newline)
s.append(repeat(' ', indentation + indentStep)) t.append(repeat(' ', indentation + indentStep))
if line.finish >= line.start: if line.finish >= line.start:
s.append(scalar[line.start .. line.finish]) t.append(scalar[line.start .. line.finish])
except: except:
var e = newException(YamlPresenterOutputError, var e = newException(YamlPresenterOutputError,
"Error while writing to output stream") "Error while writing to output stream")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
proc writeFolded(scalar: string, indentation, indentStep: int, proc writeFolded(c: Context, scalar: string, indentation, indentStep: int,
s: PresenterTarget, words: seq[tuple[start, finish: int]], words: seq[tuple[start, finish: int]],
newline: string) newline: string)
{.raises: [YamlPresenterOutputError].} = {.raises: [YamlPresenterOutputError].} =
let t = c.target
try: try:
s.append(">") t.append(">")
if scalar[^1] != '\l': s.append('-') if scalar[^1] != '\l': t.append('-')
if scalar[0] in [' ', '\t']: s.append($indentStep) if scalar[0] in [' ', '\t']: t.append($indentStep)
var curPos = 80 var curPos = 80
for word in words: for word in words:
if word.start > 0 and scalar[word.start - 1] == '\l': if word.start > 0 and scalar[word.start - 1] == '\l':
s.append(newline & newline) t.append(newline & newline)
s.append(repeat(' ', indentation + indentStep)) t.append(repeat(' ', indentation + indentStep))
curPos = indentation + indentStep curPos = indentation + indentStep
elif curPos + (word.finish - word.start) > 80: elif curPos + (word.finish - word.start) > 80:
s.append(newline) t.append(newline)
s.append(repeat(' ', indentation + indentStep)) t.append(repeat(' ', indentation + indentStep))
curPos = indentation + indentStep curPos = indentation + indentStep
else: else:
s.append(' ') t.append(' ')
curPos.inc() curPos.inc()
s.append(scalar[word.start .. word.finish]) t.append(scalar[word.start .. word.finish])
curPos += word.finish - word.start + 1 curPos += word.finish - word.start + 1
except: except:
var e = newException(YamlPresenterOutputError, var e = newException(YamlPresenterOutputError,
@ -316,106 +342,106 @@ proc writeFolded(scalar: string, indentation, indentStep: int,
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
template safeWrite(target: PresenterTarget, s: string or char) = template safeWrite(c: Context, s: string or char) =
try: target.append(s) try: c.target.append(s)
except: except:
var e = newException(YamlPresenterOutputError, "") var e = newException(YamlPresenterOutputError, "")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
proc startItem(target: PresenterTarget, style: PresentationStyle, proc startItem(c: var Context, indentation: int, isObject: bool,
indentation: int, state: var DumperState, isObject: bool,
newline: string) {.raises: [YamlPresenterOutputError].} = newline: string) {.raises: [YamlPresenterOutputError].} =
let t = c.target
try: try:
case state case c.state
of dBlockMapValue: of dBlockMapValue:
target.append(newline) t.append(newline)
target.append(repeat(' ', indentation)) t.append(repeat(' ', indentation))
if isObject or style == psCanonical: if isObject or c.options.style == psCanonical:
target.append("? ") t.append("? ")
state = dBlockExplicitMapKey c.state = dBlockExplicitMapKey
else: state = dBlockImplicitMapKey else: c.state = dBlockImplicitMapKey
of dBlockInlineMap: state = dBlockImplicitMapKey of dBlockInlineMap: c.state = dBlockImplicitMapKey
of dBlockExplicitMapKey: of dBlockExplicitMapKey:
target.append(newline) t.append(newline)
target.append(repeat(' ', indentation)) t.append(repeat(' ', indentation))
target.append(": ") t.append(": ")
state = dBlockMapValue c.state = dBlockMapValue
of dBlockImplicitMapKey: of dBlockImplicitMapKey:
target.append(": ") t.append(": ")
state = dBlockMapValue c.state = dBlockMapValue
of dFlowExplicitMapKey: of dFlowExplicitMapKey:
if style != psMinimal: if c.options.style != psMinimal:
target.append(newline) t.append(newline)
target.append(repeat(' ', indentation)) t.append(repeat(' ', indentation))
target.append(": ") t.append(": ")
state = dFlowMapValue c.state = dFlowMapValue
of dFlowMapValue: of dFlowMapValue:
if (isObject and style != psMinimal) or style in [psJson, psCanonical]: if (isObject and c.options.style != psMinimal) or c.options.style in [psJson, psCanonical]:
target.append(',' & newline & repeat(' ', indentation)) t.append(',' & newline & repeat(' ', indentation))
if style == psJson: state = dFlowImplicitMapKey if c.options.style == psJson: c.state = dFlowImplicitMapKey
else: else:
target.append("? ") t.append("? ")
state = dFlowExplicitMapKey c.state = dFlowExplicitMapKey
elif isObject and style == psMinimal: elif isObject and c.options.style == psMinimal:
target.append(", ? ") t.append(", ? ")
state = dFlowExplicitMapKey c.state = dFlowExplicitMapKey
else: else:
target.append(", ") t.append(", ")
state = dFlowImplicitMapKey c.state = dFlowImplicitMapKey
of dFlowMapStart: of dFlowMapStart:
if (isObject and style != psMinimal) or style in [psJson, psCanonical]: if (isObject and c.options.style != psMinimal) or c.options.style in [psJson, psCanonical]:
target.append(newline & repeat(' ', indentation)) t.append(newline & repeat(' ', indentation))
if style == psJson: state = dFlowImplicitMapKey if c.options.style == psJson: c.state = dFlowImplicitMapKey
else: else:
target.append("? ") t.append("? ")
state = dFlowExplicitMapKey c.state = dFlowExplicitMapKey
else: state = dFlowImplicitMapKey else: c.state = dFlowImplicitMapKey
of dFlowImplicitMapKey: of dFlowImplicitMapKey:
target.append(": ") t.append(": ")
state = dFlowMapValue c.state = dFlowMapValue
of dBlockSequenceItem: of dBlockSequenceItem:
target.append(newline) t.append(newline)
target.append(repeat(' ', indentation)) t.append(repeat(' ', indentation))
target.append("- ") t.append("- ")
of dFlowSequenceStart: of dFlowSequenceStart:
case style case c.options.style
of psMinimal, psDefault: discard of psMinimal, psDefault: discard
of psCanonical, psJson: of psCanonical, psJson:
target.append(newline) t.append(newline)
target.append(repeat(' ', indentation)) t.append(repeat(' ', indentation))
of psBlockOnly: discard # can never happen of psBlockOnly: discard # can never happen
state = dFlowSequenceItem c.state = dFlowSequenceItem
of dFlowSequenceItem: of dFlowSequenceItem:
case style case c.options.style
of psMinimal, psDefault: target.append(", ") of psMinimal, psDefault: t.append(", ")
of psCanonical, psJson: of psCanonical, psJson:
target.append(',' & newline) t.append(',' & newline)
target.append(repeat(' ', indentation)) t.append(repeat(' ', indentation))
of psBlockOnly: discard # can never happen of psBlockOnly: discard # can never happen
except: except:
var e = newException(YamlPresenterOutputError, "") var e = newException(YamlPresenterOutputError, "")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
proc writeTagAndAnchor(target: PresenterTarget, props: Properties, proc writeTagAndAnchor(c: Context, props: Properties) {.raises: [YamlPresenterOutputError].} =
tagLib: TagLibrary) {.raises: [YamlPresenterOutputError].} = let t = c.target
try: try:
if props.tag notin [yTagQuestionMark, yTagExclamationMark]: if props.tag notin [yTagQuestionMark, yTagExclamationMark]:
let tagUri = tagLib.uri(props.tag) let tagUri = c.tagLib.uri(props.tag)
let (handle, length) = tagLib.searchHandle(tagUri) let (handle, length) = c.searchHandle(tagUri)
if length > 0: if length > 0:
target.append(handle) t.append(handle)
target.append(tagUri[length..tagUri.high]) t.append(tagUri[length..tagUri.high])
target.append(' ') t.append(' ')
else: else:
target.append("!<") t.append("!<")
target.append(tagUri) t.append(tagUri)
target.append("> ") t.append("> ")
if props.anchor != yAnchorNone: if props.anchor != yAnchorNone:
target.append("&") t.append("&")
target.append($props.anchor) t.append($props.anchor)
target.append(' ') t.append(' ')
except: except:
var e = newException(YamlPresenterOutputError, "") var e = newException(YamlPresenterOutputError, "")
e.parent = getCurrentException() e.parent = getCurrentException()
@ -429,15 +455,12 @@ proc nextItem(c: var Deque, s: var YamlStream):
else: else:
result = s.next() result = s.next()
proc doPresent(s: var YamlStream, target: PresenterTarget, proc doPresent(c: var Context, s: var YamlStream) =
tagLib: TagLibrary,
options: PresentationOptions = defaultPresentationOptions) =
var var
indentation = 0 indentation = 0
levels = newSeq[DumperState]()
cached = initDeQue[Event]() cached = initDeQue[Event]()
let newline = if options.newlines == nlLF: "\l" let newline = if c.options.newlines == nlLF: "\l"
elif options.newlines == nlCRLF: "\c\l" else: "\n" elif c.options.newlines == nlCRLF: "\c\l" else: "\n"
var firstDoc = true var firstDoc = true
while true: while true:
let item = nextItem(cached, s) let item = nextItem(cached, s)
@ -445,94 +468,95 @@ proc doPresent(s: var YamlStream, target: PresenterTarget,
of yamlStartStream: discard of yamlStartStream: discard
of yamlEndStream: break of yamlEndStream: break
of yamlStartDoc: of yamlStartDoc:
resetHandles(c.handles)
for v in item.handles:
discard registerHandle(c.handles, v.handle, v.uriPrefix)
if not firstDoc: if not firstDoc:
if options.style == psJson: if c.options.style == psJson:
raise newException(YamlPresenterJsonError, raise newException(YamlPresenterJsonError,
"Cannot output more than one document in JSON style") "Cannot output more than one document in JSON style")
target.safeWrite("..." & newline) c.safeWrite("..." & newline)
if options.style != psJson: if c.options.style != psJson:
try: try:
case options.outputVersion case c.options.outputVersion
of ov1_2: target.append("%YAML 1.2" & newline) of ov1_2: c.target.append("%YAML 1.2" & newline)
of ov1_1: target.append("%YAML 1.1" & newLine) of ov1_1: c.target.append("%YAML 1.1" & newLine)
of ovNone: discard of ovNone: discard
for prefix, uri in tagLib.handles(): for v in c.handles:
if prefix == "!": if v.handle == "!":
if uri != "!": if v.uriPrefix != "!":
target.append("%TAG ! " & uri & newline) c.target.append("%TAG ! " & v.uriPrefix & newline)
elif prefix == "!!": elif v.handle == "!!":
if uri != yamlTagRepositoryPrefix: if v.uriPrefix != yamlTagRepositoryPrefix:
target.append("%TAG !! " & uri & newline) c.target.append("%TAG !! " & v.uriPrefix & newline)
else: else:
target.append("%TAG " & prefix & ' ' & uri & newline) c.target.append("%TAG " & v.handle & ' ' & v.uriPrefix & newline)
target.append("--- ") c.target.append("--- ")
except: except:
var e = newException(YamlPresenterOutputError, "") var e = newException(YamlPresenterOutputError, "")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
of yamlScalar: of yamlScalar:
if levels.len == 0: if c.levels.len == 0:
if options.style != psJson: target.safeWrite(newline) if c.options.style != psJson: c.safeWrite(newline)
else: else:
startItem(target, options.style, indentation, c.startItem(indentation, false, newline)
levels[levels.high], false, newline) if c.options.style != psJson:
if options.style != psJson: c.writeTagAndAnchor(item.scalarProperties)
writeTagAndAnchor(target, item.scalarProperties, taglib)
if options.style == psJson: if c.options.style == psJson:
let hint = guessType(item.scalarContent) let hint = guessType(item.scalarContent)
let tag = item.scalarProperties.tag let tag = item.scalarProperties.tag
if tag in [yTagQuestionMark, yTagBoolean] and if tag in [yTagQuestionMark, yTagBoolean] and
hint in {yTypeBoolTrue, yTypeBoolFalse}: hint in {yTypeBoolTrue, yTypeBoolFalse}:
target.safeWrite(if hint == yTypeBoolTrue: "true" else: "false") c.safeWrite(if hint == yTypeBoolTrue: "true" else: "false")
elif tag in [yTagQuestionMark, yTagNull] and elif tag in [yTagQuestionMark, yTagNull] and
hint == yTypeNull: hint == yTypeNull:
target.safeWrite("null") c.safeWrite("null")
elif tag in [yTagQuestionMark, yTagInteger, elif tag in [yTagQuestionMark, yTagInteger,
yTagNimInt8, yTagNimInt16, yTagNimInt32, yTagNimInt64, yTagNimInt8, yTagNimInt16, yTagNimInt32, yTagNimInt64,
yTagNimUInt8, yTagNimUInt16, yTagNimUInt32, yTagNimUInt64] and yTagNimUInt8, yTagNimUInt16, yTagNimUInt32, yTagNimUInt64] and
hint == yTypeInteger: hint == yTypeInteger:
target.safeWrite(item.scalarContent) c.safeWrite(item.scalarContent)
elif tag in [yTagQuestionMark, yTagFloat, yTagNimFloat32, elif tag in [yTagQuestionMark, yTagFloat, yTagNimFloat32,
yTagNimFloat64] and hint in {yTypeFloatInf, yTypeFloatNaN}: yTagNimFloat64] and hint in {yTypeFloatInf, yTypeFloatNaN}:
raise newException(YamlPresenterJsonError, raise newException(YamlPresenterJsonError,
"Infinity and not-a-number values cannot be presented as JSON!") "Infinity and not-a-number values cannot be presented as JSON!")
elif tag in [yTagQuestionMark, yTagFloat] and elif tag in [yTagQuestionMark, yTagFloat] and
hint == yTypeFloat: hint == yTypeFloat:
target.safeWrite(item.scalarContent) c.safeWrite(item.scalarContent)
else: writeDoubleQuotedJson(item.scalarContent, target) else: c.writeDoubleQuotedJson(item.scalarContent)
elif options.style == psCanonical: elif c.options.style == psCanonical:
writeDoubleQuoted(item.scalarContent, target, c.writeDoubleQuoted(item.scalarContent,
indentation + options.indentationStep, newline) indentation + c.options.indentationStep, newline)
else: else:
var words, lines = newSeq[tuple[start, finish: int]]() var words, lines = newSeq[tuple[start, finish: int]]()
case item.scalarContent.inspect( case item.scalarContent.inspect(
indentation + options.indentationStep, words, lines) indentation + c.options.indentationStep, words, lines)
of sLiteral: writeLiteral(item.scalarContent, indentation, of sLiteral: c.writeLiteral(item.scalarContent, indentation,
options.indentationStep, target, lines, newline) c.options.indentationStep, lines, newline)
of sFolded: writeFolded(item.scalarContent, indentation, of sFolded: c.writeFolded(item.scalarContent, indentation,
options.indentationStep, target, words, newline) c.options.indentationStep, words, newline)
of sPlain: target.safeWrite(item.scalarContent) of sPlain: c.safeWrite(item.scalarContent)
of sDoubleQuoted: writeDoubleQuoted(item.scalarContent, target, of sDoubleQuoted: c.writeDoubleQuoted(item.scalarContent,
indentation + options.indentationStep, newline) indentation + c.options.indentationStep, newline)
of yamlAlias: of yamlAlias:
if options.style == psJson: if c.options.style == psJson:
raise newException(YamlPresenterJsonError, raise newException(YamlPresenterJsonError,
"Alias not allowed in JSON output") "Alias not allowed in JSON output")
yAssert levels.len > 0 yAssert c.levels.len > 0
startItem(target, options.style, indentation, levels[levels.high], c.startItem(indentation, false, newline)
false, newline)
try: try:
target.append('*') c.target.append('*')
target.append($item.aliasTarget) c.target.append($item.aliasTarget)
except: except:
var e = newException(YamlPresenterOutputError, "") var e = newException(YamlPresenterOutputError, "")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
of yamlStartSeq: of yamlStartSeq:
var nextState: DumperState var nextState: DumperState
case options.style case c.options.style
of psDefault: of psDefault:
var length = 0 var length = 0
while true: while true:
@ -547,8 +571,7 @@ proc doPresent(s: var YamlStream, target: PresenterTarget,
break break
nextState = if length <= 60: dFlowSequenceStart else: dBlockSequenceItem nextState = if length <= 60: dFlowSequenceStart else: dBlockSequenceItem
of psJson: of psJson:
if levels.len > 0 and levels[levels.high] in if c.levels.len > 0 and c.state in [dFlowMapStart, dFlowMapValue]:
[dFlowMapStart, dFlowMapValue]:
raise newException(YamlPresenterJsonError, "Cannot have sequence as map key in JSON output!") raise newException(YamlPresenterJsonError, "Cannot have sequence as map key in JSON output!")
nextState = dFlowSequenceStart nextState = dFlowSequenceStart
of psMinimal, psCanonical: nextState = dFlowSequenceStart of psMinimal, psCanonical: nextState = dFlowSequenceStart
@ -557,33 +580,32 @@ proc doPresent(s: var YamlStream, target: PresenterTarget,
if next.kind == yamlEndSeq: nextState = dFlowSequenceStart if next.kind == yamlEndSeq: nextState = dFlowSequenceStart
else: nextState = dBlockSequenceItem else: nextState = dBlockSequenceItem
if levels.len == 0: if c.levels.len == 0:
case nextState case nextState
of dBlockSequenceItem: of dBlockSequenceItem:
if options.style != psJson: if c.options.style != psJson:
writeTagAndAnchor(target, item.seqProperties, tagLib) c.writeTagAndAnchor(item.seqProperties)
of dFlowSequenceStart: of dFlowSequenceStart:
target.safeWrite(newline) c.safeWrite(newline)
if options.style != psJson: if c.options.style != psJson:
writeTagAndAnchor(target, item.seqProperties, tagLib) c.writeTagAndAnchor(item.seqProperties)
indentation += options.indentationStep indentation += c.options.indentationStep
else: internalError("Invalid nextState: " & $nextState) else: internalError("Invalid nextState: " & $nextState)
else: else:
startItem(target, options.style, indentation, c.startItem(indentation, true, newline)
levels[levels.high], true, newline) if c.options.style != psJson:
if options.style != psJson: c.writeTagAndAnchor(item.seqProperties)
writeTagAndAnchor(target, item.seqProperties, tagLib) indentation += c.options.indentationStep
indentation += options.indentationStep
if nextState == dFlowSequenceStart: target.safeWrite('[') if nextState == dFlowSequenceStart: c.safeWrite('[')
if levels.len > 0 and options.style in [psJson, psCanonical] and if c.levels.len > 0 and c.options.style in [psJson, psCanonical] and
levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue, c.state in [dBlockExplicitMapKey, dBlockMapValue,
dBlockImplicitMapKey, dBlockSequenceItem]: dBlockImplicitMapKey, dBlockSequenceItem]:
indentation += options.indentationStep indentation += c.options.indentationStep
levels.add(nextState) c.levels.add(nextState)
of yamlStartMap: of yamlStartMap:
var nextState: DumperState var nextState: DumperState
case options.style case c.options.style
of psDefault: of psDefault:
type MapParseState = enum type MapParseState = enum
mpInitial, mpKey, mpValue, mpNeedBlock mpInitial, mpKey, mpValue, mpNeedBlock
@ -601,8 +623,7 @@ proc doPresent(s: var YamlStream, target: PresenterTarget,
of psMinimal: nextState = dFlowMapStart of psMinimal: nextState = dFlowMapStart
of psCanonical: nextState = dFlowMapStart of psCanonical: nextState = dFlowMapStart
of psJson: of psJson:
if levels.len > 0 and levels[levels.high] in if c.levels.len > 0 and c.state in [dFlowMapStart, dFlowMapValue]:
[dFlowMapStart, dFlowMapValue]:
raise newException(YamlPresenterJsonError, raise newException(YamlPresenterJsonError,
"Cannot have map as map key in JSON output!") "Cannot have map as map key in JSON output!")
nextState = dFlowMapStart nextState = dFlowMapStart
@ -610,103 +631,101 @@ proc doPresent(s: var YamlStream, target: PresenterTarget,
let next = s.peek() let next = s.peek()
if next.kind == yamlEndMap: nextState = dFlowMapStart if next.kind == yamlEndMap: nextState = dFlowMapStart
else: nextState = dBlockMapValue else: nextState = dBlockMapValue
if levels.len == 0: if c.levels.len == 0:
case nextState case nextState
of dBlockMapValue: of dBlockMapValue:
if options.style != psJson: if c.options.style != psJson:
writeTagAndAnchor(target, item.mapProperties, tagLib) c.writeTagAndAnchor(item.mapProperties)
else: else:
if options.style != psJson: if c.options.style != psJson:
target.safeWrite(newline) c.safeWrite(newline)
writeTagAndAnchor(target, item.mapProperties, tagLib) c.writeTagAndAnchor(item.mapProperties)
indentation += options.indentationStep indentation += c.options.indentationStep
of dFlowMapStart: of dFlowMapStart:
target.safeWrite(newline) c.safeWrite(newline)
if options.style != psJson: if c.options.style != psJson:
writeTagAndAnchor(target, item.mapProperties, tagLib) c.writeTagAndAnchor(item.mapProperties)
indentation += options.indentationStep indentation += c.options.indentationStep
of dBlockInlineMap: discard of dBlockInlineMap: discard
else: internalError("Invalid nextState: " & $nextState) else: internalError("Invalid nextState: " & $nextState)
else: else:
if nextState in [dBlockMapValue, dBlockImplicitMapKey]: if nextState in [dBlockMapValue, dBlockImplicitMapKey]:
startItem(target, options.style, indentation, c.startItem(indentation, true, newline)
levels[levels.high], true, newline) if c.options.style != psJson:
if options.style != psJson: c.writeTagAndAnchor(item.mapProperties)
writeTagAndAnchor(target, item.mapProperties, tagLib)
else: else:
startItem(target, options.style, indentation, c.startItem(indentation, true, newline)
levels[levels.high], true, newline) if c.options.style != psJson:
if options.style != psJson: c.writeTagAndAnchor(item.mapProperties)
writeTagAndAnchor(target, item.mapProperties, tagLib) indentation += c.options.indentationStep
indentation += options.indentationStep
if nextState == dFlowMapStart: target.safeWrite('{') if nextState == dFlowMapStart: c.safeWrite('{')
if levels.len > 0 and options.style in [psJson, psCanonical] and if c.levels.len > 0 and c.options.style in [psJson, psCanonical] and
levels[levels.high] in c.state in [dBlockExplicitMapKey, dBlockMapValue,
[dBlockExplicitMapKey, dBlockMapValue, dBlockImplicitMapKey, dBlockImplicitMapKey,
dBlockImplicitMapKey, dBlockSequenceItem]: dBlockSequenceItem]:
indentation += options.indentationStep indentation += c.options.indentationStep
levels.add(nextState) c.levels.add(nextState)
of yamlEndSeq: of yamlEndSeq:
yAssert levels.len > 0 yAssert c.levels.len > 0
case levels.pop() case c.levels.pop()
of dFlowSequenceItem: of dFlowSequenceItem:
case options.style case c.options.style
of psDefault, psMinimal, psBlockOnly: target.safeWrite(']') of psDefault, psMinimal, psBlockOnly: c.safeWrite(']')
of psJson, psCanonical: of psJson, psCanonical:
indentation -= options.indentationStep indentation -= c.options.indentationStep
try: try:
target.append(newline) c.target.append(newline)
target.append(repeat(' ', indentation)) c.target.append(repeat(' ', indentation))
target.append(']') c.target.append(']')
except: except:
var e = newException(YamlPresenterOutputError, "") var e = newException(YamlPresenterOutputError, "")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
if levels.len == 0 or levels[levels.high] notin if c.levels.len == 0 or c.state notin
[dBlockExplicitMapKey, dBlockMapValue, [dBlockExplicitMapKey, dBlockMapValue,
dBlockImplicitMapKey, dBlockSequenceItem]: dBlockImplicitMapKey, dBlockSequenceItem]:
continue continue
of dFlowSequenceStart: of dFlowSequenceStart:
if levels.len > 0 and options.style in [psJson, psCanonical] and if c.levels.len > 0 and c.options.style in [psJson, psCanonical] and
levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue, c.state in [dBlockExplicitMapKey, dBlockMapValue,
dBlockImplicitMapKey, dBlockSequenceItem]: dBlockImplicitMapKey, dBlockSequenceItem]:
indentation -= options.indentationStep indentation -= c.options.indentationStep
target.safeWrite(']') c.safeWrite(']')
of dBlockSequenceItem: discard of dBlockSequenceItem: discard
else: internalError("Invalid popped level") else: internalError("Invalid popped level")
indentation -= options.indentationStep indentation -= c.options.indentationStep
of yamlEndMap: of yamlEndMap:
yAssert levels.len > 0 yAssert c.levels.len > 0
let level = levels.pop() let level = c.levels.pop()
case level case level
of dFlowMapValue: of dFlowMapValue:
case options.style case c.options.style
of psDefault, psMinimal, psBlockOnly: target.safeWrite('}') of psDefault, psMinimal, psBlockOnly: c.safeWrite('}')
of psJson, psCanonical: of psJson, psCanonical:
indentation -= options.indentationStep indentation -= c.options.indentationStep
try: try:
target.append(newline) c.target.append(newline)
target.append(repeat(' ', indentation)) c.target.append(repeat(' ', indentation))
target.append('}') c.target.append('}')
except: except:
var e = newException(YamlPresenterOutputError, "") var e = newException(YamlPresenterOutputError, "")
e.parent = getCurrentException() e.parent = getCurrentException()
raise e raise e
if levels.len == 0 or levels[levels.high] notin if c.levels.len == 0 or c.state notin
[dBlockExplicitMapKey, dBlockMapValue, [dBlockExplicitMapKey, dBlockMapValue,
dBlockImplicitMapKey, dBlockSequenceItem]: dBlockImplicitMapKey, dBlockSequenceItem]:
continue continue
of dFlowMapStart: of dFlowMapStart:
if levels.len > 0 and options.style in [psJson, psCanonical] and if c.levels.len > 0 and c.options.style in [psJson, psCanonical] and
levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue, c.state in [dBlockExplicitMapKey, dBlockMapValue,
dBlockImplicitMapKey, dBlockSequenceItem]: dBlockImplicitMapKey, dBlockSequenceItem]:
indentation -= options.indentationStep indentation -= c.options.indentationStep
target.safeWrite('}') c.safeWrite('}')
of dBlockMapValue, dBlockInlineMap: discard of dBlockMapValue, dBlockInlineMap: discard
else: internalError("Invalid level: " & $level) else: internalError("Invalid level: " & $level)
indentation -= options.indentationStep indentation -= c.options.indentationStep
of yamlEndDoc: of yamlEndDoc:
firstDoc = false firstDoc = false
@ -716,57 +735,60 @@ proc present*(s: var YamlStream, target: Stream,
{.raises: [YamlPresenterJsonError, YamlPresenterOutputError, {.raises: [YamlPresenterJsonError, YamlPresenterOutputError,
YamlStreamError].} = YamlStreamError].} =
## Convert ``s`` to a YAML character stream and write it to ``target``. ## Convert ``s`` to a YAML character stream and write it to ``target``.
doPresent(s, target, tagLib, options) var c = Context(target: target, tagLib: tagLib, options: options)
doPresent(c, s)
proc present*(s: var YamlStream, tagLib: TagLibrary, proc present*(s: var YamlStream, tagLib: TagLibrary,
options: PresentationOptions = defaultPresentationOptions): options: PresentationOptions = defaultPresentationOptions):
string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError,
YamlStreamError].} = YamlStreamError].} =
## Convert ``s`` to a YAML character stream and return it as string. ## Convert ``s`` to a YAML character stream and return it as string.
result = ""
doPresent(s, addr result, tagLib, options)
proc doTransform(input: Stream | string, output: PresenterTarget, var
options: PresentationOptions, resolveToCoreYamlTags: bool) = ss = newStringStream()
c = Context(target: ss, tagLib: tagLib, options: options)
doPresent(c, s)
return ss.data
proc doTransform(c: var Context, input: Stream,
resolveToCoreYamlTags: bool) =
var var
taglib = initExtendedTagLibrary() taglib = initExtendedTagLibrary()
parser: YamlParser parser: YamlParser
parser.init(tagLib) parser.init(tagLib)
var events = parser.parse(input) var events = parser.parse(input)
try: try:
if options.style == psCanonical: if c.options.style == psCanonical:
var bys: YamlStream = newBufferYamlStream() var bys: YamlStream = newBufferYamlStream()
for e in events: for e in events:
if resolveToCoreYamlTags: if resolveToCoreYamlTags:
var event = e var event = e
case event.kind case event.kind
of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq: of yamlStartStream, yamlEndStream, yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq:
discard discard
of yamlStartMap: of yamlStartMap:
if event.mapTag in [yTagQuestionMark, yTagExclamationMark]: if event.mapProperties.tag in [yTagQuestionMark, yTagExclamationMark]:
event.mapTag = yTagMapping event.mapProperties.tag = yTagMapping
of yamlStartSeq: of yamlStartSeq:
if event.seqTag in [yTagQuestionMark, yTagExclamationMark]: if event.seqProperties.tag in [yTagQuestionMark, yTagExclamationMark]:
event.seqTag = yTagSequence event.seqProperties.tag = yTagSequence
of yamlScalar: of yamlScalar:
if event.scalarTag == yTagQuestionMark: if event.scalarProperties.tag == yTagQuestionMark:
case guessType(event.scalarContent) case guessType(event.scalarContent)
of yTypeInteger: event.scalarTag = yTagInteger of yTypeInteger: event.scalarProperties.tag = yTagInteger
of yTypeFloat, yTypeFloatInf, yTypeFloatNaN: of yTypeFloat, yTypeFloatInf, yTypeFloatNaN:
event.scalarTag = yTagFloat event.scalarProperties.tag = yTagFloat
of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean of yTypeBoolTrue, yTypeBoolFalse: event.scalarProperties.tag = yTagBoolean
of yTypeNull: event.scalarTag = yTagNull of yTypeNull: event.scalarProperties.tag = yTagNull
of yTypeTimestamp: event.scalarTag = yTagTimestamp of yTypeTimestamp: event.scalarProperties.tag = yTagTimestamp
of yTypeUnknown: event.scalarTag = yTagString of yTypeUnknown: event.scalarProperties.tag = yTagString
elif event.scalarTag == yTagExclamationMark: elif event.scalarProperties.tag == yTagExclamationMark:
event.scalarTag = yTagString event.scalarProperties.tag = yTagString
BufferYamlStream(bys).put(event) BufferYamlStream(bys).put(event)
else: BufferYamlStream(bys).put(e) else: BufferYamlStream(bys).put(e)
when output is ptr[string]: output[] = present(bys, tagLib, options) doPresent(c, bys)
else: present(bys, output, tagLib, options)
else: else:
when output is ptr[string]: output[] = present(events, tagLib, options) doPresent(c, events)
else: present(events, output, tagLib, options)
except YamlStreamError: except YamlStreamError:
var e = getCurrentException() var e = getCurrentException()
while e.parent of YamlStreamError: e = e.parent while e.parent of YamlStreamError: e = e.parent
@ -774,6 +796,9 @@ proc doTransform(input: Stream | string, output: PresenterTarget,
elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent) elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent)
else: internalError("Unexpected exception: " & e.parent.repr) else: internalError("Unexpected exception: " & e.parent.repr)
proc genInput(input: Stream): Stream = input
proc genInput(input: string): Stream = newStringStream(input)
proc transform*(input: Stream | string, output: Stream, proc transform*(input: Stream | string, output: Stream,
options: PresentationOptions = defaultPresentationOptions, options: PresentationOptions = defaultPresentationOptions,
resolveToCoreYamlTags: bool = false) resolveToCoreYamlTags: bool = false)
@ -783,7 +808,7 @@ proc transform*(input: Stream | string, output: Stream,
## while resolving non-specific tags to the ones in the YAML core tag ## while resolving non-specific tags to the ones in the YAML core tag
## library. If ``resolveToCoreYamlTags`` is ``true``, non-specific tags will ## library. If ``resolveToCoreYamlTags`` is ``true``, non-specific tags will
## be replaced by specific tags according to the YAML core schema. ## be replaced by specific tags according to the YAML core schema.
doTransform(input, output, options, resolveToCoreYamlTags) doTransform(genInput(input), output, options, resolveToCoreYamlTags)
proc transform*(input: Stream | string, proc transform*(input: Stream | string,
options: PresentationOptions = defaultPresentationOptions, options: PresentationOptions = defaultPresentationOptions,
@ -796,4 +821,4 @@ proc transform*(input: Stream | string,
## ``true``, non-specific tags will be replaced by specific tags according to ## ``true``, non-specific tags will be replaced by specific tags according to
## the YAML core schema. ## the YAML core schema.
result = "" result = ""
doTransform(input, addr result, options, resolveToCoreYamlTags) doTransform(genInput(input), addr result, options, resolveToCoreYamlTags)

15
yaml/private/internal.nim
View File

@ -61,4 +61,17 @@ proc nextAnchor*(s: var string, i: int) =
s[i] = 'a' s[i] = 'a'
nextAnchor(s, i - 1) nextAnchor(s, i - 1)
else: else:
inc(s[i]) inc(s[i])
template resetHandles*(handles: var seq[tuple[handle, uriPrefix: string]]) {.dirty.} =
handles.setLen(0)
handles.add(("!", "!"))
handles.add(("!!", yamlTagRepositoryPrefix))
proc registerHandle*(handles: var seq[tuple[handle, uriPrefix: string]], handle, uriPrefix: string): bool =
for i in countup(0, len(handles)-1):
if handles[i].handle == handle:
handles[i].uriPrefix = uriPrefix
return false
handles.add((handle, uriPrefix))
return false

35
yaml/serialization.nim
View File

@ -1361,15 +1361,11 @@ proc construct*[T](s: var YamlStream, target: var T)
var context = newConstructionContext() var context = newConstructionContext()
try: try:
var e = s.next() var e = s.next()
yAssert(e.kind == yamlStartStream)
e = s.next()
yAssert(e.kind == yamlStartDoc) yAssert(e.kind == yamlStartDoc)
constructChild(s, context, target) constructChild(s, context, target)
e = s.next() e = s.next()
yAssert(e.kind == yamlEndDoc) yAssert(e.kind == yamlEndDoc)
e = s.next()
yAssert(e.kind == yamlEndStream)
except YamlConstructionError: except YamlConstructionError:
raise (ref YamlConstructionError)(getCurrentException()) raise (ref YamlConstructionError)(getCurrentException())
except YamlStreamError: except YamlStreamError:
@ -1386,10 +1382,19 @@ proc construct*[T](s: var YamlStream, target: var T)
proc load*[K](input: Stream | string, target: var K) proc load*[K](input: Stream | string, target: var K)
{.raises: [YamlConstructionError, IOError, YamlParserError].} = {.raises: [YamlConstructionError, IOError, YamlParserError].} =
## Loads a Nim value from a YAML character stream. ## Loads a Nim value from a YAML character stream.
var parser: YamlParser var
parser.init(serializationTagLibrary) parser = initYamlParser(serializationTagLibrary)
var events = parser.parse(input) events = parser.parse(input)
try: construct(events, target) try:
var e = events.next()
yAssert(e.kind == yamlStartStream)
construct(events, target)
e = events.next()
if e.kind != yamlEndStream:
var ex = (ref YamlConstructionError)(
mark: e.startPos, msg: "stream contains multiple document")
discard events.getLastTokenContext(ex.lineContent)
raise ex
except YamlStreamError: except YamlStreamError:
let e = (ref YamlStreamError)(getCurrentException()) let e = (ref YamlStreamError)(getCurrentException())
if e.parent of IOError: raise (ref IOError)(e.parent) if e.parent of IOError: raise (ref IOError)(e.parent)
@ -1397,16 +1402,18 @@ proc load*[K](input: Stream | string, target: var K)
else: internalError("Unexpected exception: " & $e.parent.name) else: internalError("Unexpected exception: " & $e.parent.name)
proc loadMultiDoc*[K](input: Stream | string, target: var seq[K]) = proc loadMultiDoc*[K](input: Stream | string, target: var seq[K]) =
var parser: YamlParser var
parser.init(serializationTagLibrary) parser = initYamlParser(serializationTagLibrary)
var events = parser.parse(input) events = parser.parse(input)
discard events.next() # stream start e = events.next()
yAssert(e.kind == yamlStartStream)
try: try:
while events.peek().kind == yamlStartDoc: while events.peek().kind == yamlStartDoc:
var item: K var item: K
construct(events, item) construct(events, item)
target.add(item) target.add(item)
discard events.next() # stream end e = events.next()
yAssert(e.kind == yamlEndStream)
except YamlConstructionError: except YamlConstructionError:
var e = (ref YamlConstructionError)(getCurrentException()) var e = (ref YamlConstructionError)(getCurrentException())
discard events.getLastTokenContext(e.lineContent) discard events.getLastTokenContext(e.lineContent)
@ -1433,7 +1440,7 @@ proc represent*[T](value: T, ts: TagStyle = tsRootOnly,
bys.put(e) bys.put(e)
) )
bys.put(startStreamEvent()) bys.put(startStreamEvent())
bys.put(startDocEvent()) bys.put(startDocEvent(handles = @[("!n!", nimyamlTagRepositoryPrefix)]))
representChild(value, ts, context) representChild(value, ts, context)
bys.put(endDocEvent()) bys.put(endDocEvent())
bys.put(endStreamEvent()) bys.put(endStreamEvent())

52
yaml/taglib.nim
View File

@ -12,7 +12,7 @@
## and create own tags. It also enables you to define tags for types used with ## and create own tags. It also enables you to define tags for types used with
## the serialization API. ## the serialization API.
import tables, macros, hashes, strutils import tables, macros, hashes
import data import data
type type
@ -30,20 +30,14 @@ type
## `initExtendedTagLibrary <#initExtendedTagLibrary>`_. ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_.
tags*: Table[string, TagId] tags*: Table[string, TagId]
nextCustomTagId*: TagId nextCustomTagId*: TagId
prefixes: seq[tuple[prefix, uri: string]]
proc initTagLibrary*(): TagLibrary {.raises: [].} = proc initTagLibrary*(): TagLibrary {.raises: [].} =
## initializes the ``tags`` table and sets ``nextCustomTagId`` to ## initializes the ``tags`` table and sets ``nextCustomTagId`` to
## ``yFirstCustomTagId``. ## ``yFirstCustomTagId``.
new(result) new(result)
result.tags = initTable[string, TagId]() result.tags = initTable[string, TagId]()
result.prefixes = @[("!", "!"), ("!!", yamlTagRepositoryPrefix)]
result.nextCustomTagId = yFirstCustomTagId
proc resetPrefixes*(tagLib: TagLibrary) {.raises: [].} = result.nextCustomTagId = yFirstCustomTagId
## resets the registered prefixes in the given tag library, so that it
## only contains the prefixes `!` and `!!`.
tagLib.prefixes.setLen(2)
proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} = proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} =
## registers a custom tag URI with a ``TagLibrary``. The URI will get ## registers a custom tag URI with a ``TagLibrary``. The URI will get
@ -108,7 +102,6 @@ proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} =
proc initSerializationTagLibrary*(): TagLibrary = proc initSerializationTagLibrary*(): TagLibrary =
result = initTagLibrary() result = initTagLibrary()
result.prefixes.add(("!n!", nimyamlTagRepositoryPrefix))
result.tags["!"] = yTagExclamationMark result.tags["!"] = yTagExclamationMark
result.tags["?"] = yTagQuestionMark result.tags["?"] = yTagQuestionMark
result.tags[y"str"] = yTagString result.tags[y"str"] = yTagString
@ -195,47 +188,6 @@ setTagUri(uint64, n"system:uint64", yTagNimUInt64)
setTagUri(float32, n"system:float32", yTagNimFloat32) setTagUri(float32, n"system:float32", yTagNimFloat32)
setTagUri(float64, n"system:float64", yTagNimFloat64) setTagUri(float64, n"system:float64", yTagNimFloat64)
proc registerHandle*(tagLib: TagLibrary, uri, prefix: string): bool =
## Registers a handle for a prefix. When presenting any tag that starts with
## this prefix, the handle is used instead. Also causes the presenter to
## output a TAG directive for the handle.
##
## Returns true iff a new item has been created, false if an existing item
## has been updated.
for i in countup(0, len(tagLib.prefixes)-1):
if tagLib.prefixes[i].prefix == prefix:
tagLib.prefixes[i].uri = uri
return false
taglib.prefixes.add((prefix, uri))
return false
proc searchHandle*(tagLib: TagLibrary, tag: string):
tuple[handle: string, len: int] {.raises: [].} =
## search in the registered tag handles for one whose prefix matches the start
## of the given tag. If multiple registered handles match, the one with the
## longest prefix is returned. If no registered handle matches, (nil, 0) is
## returned.
result.len = 0
for item in tagLib.prefixes:
if item.uri.len > result.len:
if tag.startsWith(item.uri):
result.len = item.uri.len
result.handle = item.prefix
proc resolve*(tagLib: TagLibrary, handle: string): string {.raises: [].} =
## try to resolve the given tag handle.
## return the registered URI if the tag handle is found.
## if the handle is unknown, return the empty string.
for item in tagLib.prefixes:
if item.prefix == handle:
return item.uri
return ""
iterator handles*(tagLib: TagLibrary): tuple[prefix, uri: string] =
## iterate over registered tag handles that may be used as shortcuts
## (e.g. ``!n!`` for ``tag:nimyaml.org,2016:``)
for item in tagLib.prefixes.items(): yield item
proc nimTag*(suffix: string): string = proc nimTag*(suffix: string): string =
## prepends NimYAML's tag repository prefix to the given suffix. For example, ## prepends NimYAML's tag repository prefix to the given suffix. For example,
## ``nimTag("system:char")`` yields ``"tag:nimyaml.org,2016:system:char"``. ## ``nimTag("system:char")`` yields ``"tag:nimyaml.org,2016:system:char"``.