Merge branch 'devel'

This commit is contained in:
Felix Krause 2016-12-15 18:25:08 +01:00
commit 55f423b00a
15 changed files with 391 additions and 135 deletions

3
.gitignore vendored
View File

@ -5,7 +5,6 @@ test/tdom
test/tserialization
test/tjson
test/tparser
test/yamlTestSuite
test/tquickstart
test/*.exe
test/*.pdb
@ -22,5 +21,5 @@ docout
doc/rstPreproc
doc/tmp.rst
doc/**/code
yaml-dev-kit
test/yaml-test-suite
nimsuggest.log

View File

@ -1,3 +1,19 @@
### 0.9.0
Features:
* Better DOM API:
- yMapping is now a Table
- field names have changed to imitate those of Nim's json API
- Better getter and setter procs
* Added ability to resolve non-specific tags in presenter.transform
Bugfixes:
* Fixed parsing floating point literals (#30)
* Fixed a bug with variant records (#31)
* Empty documents now always contain an empty scalar
### 0.8.0
Features:

View File

@ -74,7 +74,7 @@ updated as you type.
var params = "style=" + encodeURIComponent(document.querySelector(
"input[name=style]:checked").value) + "&input=" + encodeURIComponent(
document.getElementById("yaml-input").value);
r.open("POST", "http://flyx.org:5000", true);
r.open("POST", "https://nimyaml.org/webservice/", true);
r.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
r.onreadystatechange = function() {
if (r.readyState == 4) {

View File

@ -897,6 +897,7 @@ proc blockScalar[T](lex: YamlLexer): bool =
break
else:
lex.blockScalarIndent += lex.indentation
lex.indentation = 0
if lex.c notin {'.', '-'} or lex.indentation == 0:
if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer
else:

View File

@ -47,7 +47,7 @@ routes:
var
output = newStringStream()
highlighted = ""
transform(newStringStream(@"input"), output, defineOptions(style))
transform(newStringStream(@"input"), output, defineOptions(style), true)
# syntax highlighting (stolen and modified from stlib's rstgen)
var g: GeneralTokenizer

View File

@ -5,7 +5,7 @@
# distribution, for details about the copyright.
import "../yaml"
import unittest, commonTestUtils, streams
import unittest, commonTestUtils, streams, tables
suite "DOM":
test "Composing simple Scalar":
@ -25,13 +25,13 @@ suite "DOM":
result = loadDOM(input)
assert result.root.kind == ySequence
assert result.root.tag == "?"
assert result.root.children.len == 2
assert result.root.children[0].kind == yScalar
assert result.root.children[0].tag == "tag:yaml.org,2002:str"
assert result.root.children[0].content == "a"
assert result.root.children[1].kind == yScalar
assert result.root.children[1].tag == "tag:yaml.org,2002:bool"
assert result.root.children[1].content == "no"
assert result.root.len == 2
assert result.root[0].kind == yScalar
assert result.root[0].tag == "tag:yaml.org,2002:str"
assert result.root[0].content == "a"
assert result.root[1].kind == yScalar
assert result.root[1].tag == "tag:yaml.org,2002:bool"
assert result.root[1].content == "no"
test "Serializing sequence":
let input = initYamlDoc(newYamlNode([
newYamlNode("a", "tag:yaml.org,2002:str"),
@ -46,12 +46,13 @@ suite "DOM":
result = loadDOM(input)
assert result.root.kind == yMapping
assert result.root.tag == "tag:yaml.org,2002:map"
assert result.root.pairs.len == 1
assert result.root.pairs[0].key.kind == yScalar
assert result.root.pairs[0].key.tag == "!foo"
assert result.root.pairs[0].key.content == "bar"
assert result.root.pairs[0].value.kind == ySequence
assert result.root.pairs[0].value.children.len == 2
assert result.root.fields.len == 1
for key, value in result.root.fields.pairs:
assert key.kind == yScalar
assert key.tag == "!foo"
assert key.content == "bar"
assert value.kind == ySequence
assert value.len == 2
test "Serializing mapping":
let input = initYamlDoc(newYamlNode([
(key: newYamlNode("bar"), value: newYamlNode([newYamlNode("a"),
@ -65,13 +66,13 @@ suite "DOM":
input = newStringStream("- &a foo\n- &b bar\n- *a\n- *b")
result = loadDOM(input)
assert result.root.kind == ySequence
assert result.root.children.len == 4
assert result.root.children[0].kind == yScalar
assert result.root.children[0].content == "foo"
assert result.root.children[1].kind == yScalar
assert result.root.children[1].content == "bar"
assert result.root.children[0] == result.root.children[2]
assert result.root.children[1] == result.root.children[3]
assert result.root.len == 4
assert result.root[0].kind == yScalar
assert result.root[0].content == "foo"
assert result.root[1].kind == yScalar
assert result.root[1].content == "bar"
assert cast[pointer](result.root[0]) == cast[pointer](result.root[2])
assert cast[pointer](result.root[1]) == cast[pointer](result.root[3])
test "Serializing with anchors":
let
a = newYamlNode("a")

View File

@ -5,12 +5,13 @@
# distribution, for details about the copyright.
import "../yaml"
import lexbase, streams, tables
import lexbase, streams, tables, strutils
type
LexerToken = enum
plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, minusSeq,
eqVal, eqAli, chevTag, andAnchor, quotContent, colonContent, noToken
eqVal, eqAli, chevTag, andAnchor, starAnchor, quotContent, colonContent,
explDirEnd, explDocEnd, noToken
StreamPos = enum
beforeStream, inStream, afterStream
@ -29,7 +30,7 @@ proc nextToken(lex: var EventLexer): LexerToken =
else: break
if lex.buf[lex.bufpos] == EndOfFile: return noToken
case lex.buf[lex.bufpos]
of ':', '"':
of ':', '"', '\'', '|', '>':
let t = if lex.buf[lex.bufpos] == ':': colonContent else: quotContent
lex.content = ""
lex.bufpos.inc()
@ -73,6 +74,13 @@ proc nextToken(lex: var EventLexer): LexerToken =
lex.content.add(lex.buf[lex.bufpos])
lex.bufpos.inc()
result = andAnchor
of '*':
lex.content = ""
lex.bufpos.inc()
while lex.buf[lex.bufpos] notin {' ', '\t', '\r', '\l', EndOfFile}:
lex.content.add(lex.buf[lex.bufpos])
lex.bufpos.inc()
result = starAnchor
else:
lex.content = ""
while lex.buf[lex.bufpos] notin {' ', '\t', '\r', '\l', EndOfFile}:
@ -89,6 +97,8 @@ proc nextToken(lex: var EventLexer): LexerToken =
of "-SEQ": result = minusSeq
of "=VAL": result = eqVal
of "=ALI": result = eqAli
of "---": result = explDirEnd
of "...": result = explDocEnd
else: raise newException(EventStreamError, "Invalid token: " & lex.content)
template assertInStream() {.dirty.} =
@ -210,12 +220,19 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream =
if curAnchor() != yAnchorNone:
raise newException(EventStreamError,
"Duplicate anchor in " & $curEvent.kind)
if curEvent.kind == yamlAlias:
curEvent.aliasTarget = anchors[lex.content]
anchors[lex.content] = nextAnchorId
setCurAnchor(nextAnchorId)
nextAnchorId = (AnchorId)(((int)nextAnchorId) + 1)
of starAnchor:
assertInEvent("alias")
if curEvent.kind != yamlAlias:
raise newException(EventStreamError, "Unexpected alias: " &
escape(lex.content))
elif curEvent.aliasTarget != yAnchorNone:
raise newException(EventStreamError, "Duplicate alias target: " &
escape(lex.content))
else:
anchors[lex.content] = nextAnchorId
setCurAnchor(nextAnchorId)
nextAnchorId = (AnchorId)(((int)nextAnchorId) + 1)
curEvent.aliasTarget = anchors[lex.content]
of quotContent:
assertInEvent("scalar content")
if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark)
@ -229,5 +246,14 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream =
if curEvent.kind != yamlScalar:
raise newException(EventStreamError,
"scalar content in non-scalar tag")
of explDirEnd:
assertInEvent("explicit directives end")
if curEvent.kind != yamlStartDoc:
raise newException(EventStreamError,
"Unexpected explicit directives end")
of explDocEnd:
if curEvent.kind != yamlEndDoc:
raise newException(EventStreamError,
"Unexpected explicit document end")
of noToken: discard
result = initYamlStream(backend)

View File

@ -8,19 +8,21 @@ import os, osproc, terminal, strutils, streams, macros, unittest
import testEventParser, commonTestUtils
import "../yaml"
const devKitFolder = "yaml-dev-kit"
const
testSuiteFolder = "yaml-test-suite"
testSuiteUrl = "https://github.com/yaml/yaml-test-suite.git"
proc echoError(msg: string) =
styledWriteLine(stdout, fgRed, "[error] ", fgWhite, msg, resetStyle)
proc ensureDevKitCloneCorrect(pwd: string) {.compileTime.} =
let absolutePath = pwd / devKitFolder
proc ensureTestSuiteCloneCorrect(pwd: string) {.compileTime.} =
let absolutePath = pwd / testSuiteFolder
if dirExists(absolutePath):
var isCorrectClone = true
if dirExists(absolutePath / ".git"):
let remoteUrl =
staticExec("cd \"" & absolutePath & "\" && git remote get-url origin")
if remoteUrl != "https://github.com/ingydotnet/yaml-dev-kit.git":
if remoteUrl != testSuiteUrl:
isCorrectClone = false
let branches = staticExec("cd \"" & absolutePath & "\" && git branch")
if "* data" notin branches.splitLines():
@ -28,25 +30,25 @@ proc ensureDevKitCloneCorrect(pwd: string) {.compileTime.} =
if isCorrectClone:
let updateOutput = staticExec("cd \"" & absolutePath & "\" && git pull")
#if uError != 0:
# echo "could not update yaml-dev-kit! please fix this problem and compile again."
# echo "could not update yaml-test-suite! please fix this problem and compile again."
# echo "output:\n"
# echo "$ git pull"
# echo updateOutput
# quit 1
else:
echo devKitFolder, " exists, but is not in expected state. Make sure it is a git repo,"
echo "cloned from https://github.com/ingydotnet/yaml-dev-kit.git, and the data branch"
echo "is active. Alternatively, delete the folder " & devKitFolder & '.'
echo testSuiteFolder, " exists, but is not in expected state. Make sure it is a git repo,"
echo "cloned from ", testSuiteUrl, ", and the data branch"
echo "is active. Alternatively, delete the folder " & testSuiteFolder & '.'
quit 1
else:
let cloneOutput = staticExec("cd \"" & pwd &
"\" && git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data")
"\" && git clone " & testSuiteUrl & " -b data")
#if cError != 0:
if not(dirExists(absolutePath)) or not(dirExists(absolutePath / ".git")) or
not(dirExists(absolutePath / "229Q")):
echo "could not clone https://github.com/ingydotnet/yaml-dev-kit.git. Make sure"
echo "could not clone ", testSuiteUrl, ". Make sure"
echo "you are connected to the internet and your proxy settings are correct. output:\n"
echo "$ git clone https://github.com/ingydotnet/yaml-dev-kit.git"
echo "$ git clone ", testSuiteUrl, " -b data"
echo cloneOutput
quit 1
@ -95,9 +97,9 @@ proc parserTest(path: string): bool =
macro genTests(): untyped =
let
pwd = staticExec("pwd")
absolutePath = '"' & (pwd / devKitFolder) & '"'
absolutePath = '"' & (pwd / testSuiteFolder) & '"'
echo "[tparser] Generating tests from " & absolutePath
ensureDevKitCloneCorrect(pwd)
ensureTestSuiteCloneCorrect(pwd)
result = newStmtList()
# walkDir for some crude reason does not work with travis build
let dirItems = staticExec("ls -1d " & absolutePath / "*")
@ -108,6 +110,6 @@ macro genTests(): untyped =
newLit(strip(title) & " [" &
dirPath[^4..^1] & ']'), newCall("doAssert", newCall("parserTest",
newLit(dirPath)))))
result = newCall("suite", newLit("Parser Tests (from yaml-dev-kit)"), result)
result = newCall("suite", newLit("Parser Tests (from yaml-test-suite)"), result)
genTests()

View File

@ -5,7 +5,7 @@
# distribution, for details about the copyright.
import "../yaml"
import unittest, strutils, streams, tables, times
import unittest, strutils, streams, tables, times, math
type
MyTuple = tuple
@ -35,8 +35,7 @@ type
case kind: AnimalKind
of akCat:
purringIntensity: int
of akDog:
barkometer: int
of akDog: barkometer: int
DumbEnum = enum
deA, deB, deC, deD
@ -197,6 +196,15 @@ suite "Serialization":
load(input, result)
assert(result == 14)
test "Load floats":
let input = "[6.8523015e+5, 685.230_15e+03, 685_230.15, -.inf, .NaN]"
var result: seq[float]
load(input, result)
for i in 0..2:
assert result[i] == 6.8523015e+5
assert result[3] == NegInf
assert classify(result[4]) == fcNan
test "Load nil string":
let input = newStringStream("!<tag:nimyaml.org,2016:nil:string> \"\"")
var result: string

View File

@ -1,6 +1,6 @@
# Package
version = "0.8.0"
version = "0.9.0"
author = "Felix Krause"
description = "YAML 1.2 implementation for Nim"
license = "MIT"

View File

@ -12,11 +12,14 @@
## structure. It can also dump the structure back to YAML. Formally, it
## represents the *Representation Graph* as defined in the YAML specification.
##
## The main interface of this API are ``loadDOM`` and ``dumpDOM``. The other
## The main interface of this API are ``loadDom`` and ``dumpDom``. The other
## exposed procs are low-level and useful if you want to load or generate parts
## of a ``YamlStream``.
##
## The ``YamlNode`` objects in the DOM can be used similarly to the ``JsonNode``
## objects of Nim's `json module <http://nim-lang.org/docs/json.html>`_.
import tables, streams
import tables, streams, hashes, sets, strutils
import stream, taglib, serialization, ../private/internal, parser,
presenter
@ -31,23 +34,98 @@ type
tag*: string
case kind*: YamlNodeKind
of yScalar: content*: string
of ySequence: children*: seq[YamlNode]
of yMapping: pairs*: seq[tuple[key, value: YamlNode]]
of ySequence: elems*: seq[YamlNode]
of yMapping: fields*: TableRef[YamlNode, YamlNode]
# compiler does not like Table[YamlNode, YamlNode]
YamlDocument* = object
## Represents a YAML document.
root*: YamlNode
proc hash*(o: YamlNode): Hash =
result = o.tag.hash
case o.kind
of yScalar: result = result !& o.content.hash
of yMapping:
for key, value in o.fields.pairs:
result = result !& key.hash !& value.hash
of ySequence:
for item in o.elems:
result = result !& item.hash
result = !$result
proc eqImpl(x, y: YamlNode, alreadyVisited: var HashSet[pointer]): bool =
template compare(a, b: YamlNode) {.dirty.} =
if cast[pointer](a) != cast[pointer](b):
if cast[pointer](a) in alreadyVisited and
cast[pointer](b) in alreadyVisited:
# prevent infinite loop!
return false
elif a != b: return false
if x.kind != y.kind or x.tag != y.tag: return false
alreadyVisited.incl(cast[pointer](x))
alreadyVisited.incl(cast[pointer](y))
case x.kind
of yScalar: result = x.content == y.content
of ySequence:
if x.elems.len != y.elems.len: return false
for i in 0..<x.elems.len:
compare(x.elems[i], y.elems[i])
of yMapping:
if x.fields.len != y.fields.len: return false
for xKey, xValue in x.fields.pairs:
let xKeyVisited = cast[pointer](xKey) in alreadyVisited
var matchingValue: ref YamlNodeObj = nil
for yKey, yValue in y.fields.pairs:
if cast[pointer](yKey) != cast[pointer](xKey):
if cast[pointer](yKey) in alreadyVisited and xKeyVisited:
# prevent infinite loop!
continue
if xKey == yKey:
matchingValue = yValue
break
else:
matchingValue = yValue
break
if isNil(matchingValue): return false
compare(xValue, matchingValue)
proc `==`*(x, y: YamlNode): bool =
var alreadyVisited = initSet[pointer]()
result = eqImpl(x, y, alreadyVisited)
proc `$`*(n: YamlNode): string =
result = "!<" & n.tag & "> "
case n.kind
of yScalar: result.add(escape(n.content))
of ySequence:
result.add('[')
for item in n.elems:
result.add($item)
result.add(", ")
result.setLen(result.len - 1)
result[^1] = ']'
of yMapping:
result.add('{')
for key, value in n.fields.pairs:
result.add($key)
result.add(": ")
result.add($value)
result.add(", ")
result.setLen(result.len - 1)
result[^1] = '}'
proc newYamlNode*(content: string, tag: string = "?"): YamlNode =
YamlNode(kind: yScalar, content: content, tag: tag)
proc newYamlNode*(children: openarray[YamlNode], tag: string = "?"):
proc newYamlNode*(elems: openarray[YamlNode], tag: string = "?"):
YamlNode =
YamlNode(kind: ySequence, children: @children, tag: tag)
YamlNode(kind: ySequence, elems: @elems, tag: tag)
proc newYamlNode*(pairs: openarray[tuple[key, value: YamlNode]],
proc newYamlNode*(fields: openarray[(YamlNode, YamlNode)],
tag: string = "?"): YamlNode =
YamlNode(kind: yMapping, pairs: @pairs, tag: tag)
YamlNode(kind: yMapping, fields: newTable(fields), tag: tag)
proc initYamlDoc*(root: YamlNode): YamlDocument = result.root = root
@ -62,12 +140,14 @@ proc composeNode(s: var YamlStream, tagLib: TagLibrary,
of yamlStartMap:
result.tag = tagLib.uri(start.mapTag)
result.kind = yMapping
result.pairs = newSeq[tuple[key, value: YamlNode]]()
result.fields = newTable[YamlNode, YamlNode]()
while s.peek().kind != yamlEndMap:
let
key = composeNode(s, tagLib, c)
value = composeNode(s, tagLib, c)
result.pairs.add((key: key, value: value))
if result.fields.hasKeyOrPut(key, value):
raise newException(YamlConstructionError,
"Duplicate key: " & $key)
discard s.next()
if start.mapAnchor != yAnchorNone:
yAssert(not c.refs.hasKey(start.mapAnchor))
@ -75,9 +155,9 @@ proc composeNode(s: var YamlStream, tagLib: TagLibrary,
of yamlStartSeq:
result.tag = tagLib.uri(start.seqTag)
result.kind = ySequence
result.children = newSeq[YamlNode]()
result.elems = newSeq[YamlNode]()
while s.peek().kind != yamlEndSeq:
result.children.add(composeNode(s, tagLib, c))
result.elems.add(composeNode(s, tagLib, c))
if start.seqAnchor != yAnchorNone:
yAssert(not c.refs.hasKey(start.seqAnchor))
c.refs[start.seqAnchor] = cast[pointer](result)
@ -106,7 +186,7 @@ proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument
n = s.next()
yAssert n.kind == yamlEndDoc
proc loadDOM*(s: Stream | string): YamlDocument
proc loadDom*(s: Stream | string): YamlDocument
{.raises: [IOError, YamlParserError, YamlConstructionError].} =
var
tagLib = initExtendedTagLibrary()
@ -148,14 +228,14 @@ proc serializeNode(n: YamlNode, c: SerializationContext, a: AnchorStyle,
of yScalar: c.put(scalarEvent(n.content, tagId, anchor))
of ySequence:
c.put(startSeqEvent(tagId, anchor))
for item in n.children:
for item in n.elems:
serializeNode(item, c, a, tagLib)
c.put(endSeqEvent())
of yMapping:
c.put(startMapEvent(tagId, anchor))
for i in n.pairs:
serializeNode(i.key, c, a, tagLib)
serializeNode(i.value, c, a, tagLib)
for key, value in n.fields.pairs:
serializeNode(key, c, a, tagLib)
serializeNode(value, c, a, tagLib)
c.put(endMapEvent())
template processAnchoredEvent(target: untyped, c: SerializationContext): typed =
@ -182,7 +262,7 @@ proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy):
else: discard
result = bys
proc dumpDOM*(doc: YamlDocument, target: Stream,
proc dumpDom*(doc: YamlDocument, target: Stream,
anchorStyle: AnchorStyle = asTidy,
options: PresentationOptions = defaultPresentationOptions)
{.raises: [YamlPresenterJsonError, YamlPresenterOutputError,
@ -193,3 +273,70 @@ proc dumpDOM*(doc: YamlDocument, target: Stream,
events = serialize(doc, tagLib,
if options.style == psJson: asNone else: anchorStyle)
present(events, target, tagLib, options)
proc `[]`*(node: YamlNode, i: int): YamlNode =
## Get the node at index *i* from a sequence. *node* must be a *ySequence*.
assert node.kind == ySequence
node.elems[i]
proc `[]=`*(node: var YamlNode, i: int, val: YamlNode) =
## Set the node at index *i* of a sequence. *node* must be a *ySequence*.
assert node.kind == ySequence
node.elems[i] = val
proc `[]`*(node: YamlNode, key: YamlNode): YamlNode =
## Get the value for a key in a mapping. *node* must be a *yMapping*.
assert node.kind == yMapping
node.fields[key]
proc `[]=`*(node: YamlNode, key: YamlNode, value: YamlNode) =
## Set the value for a key in a mapping. *node* must be a *yMapping*.
node.fields[key] = value
proc `[]`*(node: YamlNode, key: string): YamlNode =
## Get the value for a string key in a mapping. *node* must be a *yMapping*.
## This searches for a scalar key with content *key* and either no explicit
## tag or the explicit tag ``!!str``.
assert node.kind == yMapping
var keyNode = YamlNode(kind: yScalar, tag: "!", content: key)
result = node.fields.getOrDefault(keyNode)
if isNil(result):
keyNode.tag = "?"
result = node.fields.getOrDefault(keyNode)
if isNil(result):
keyNode.tag = nimTag(yamlTagRepositoryPrefix & "str")
result = node.fields.getOrDefault(keyNode)
if isNil(result):
raise newException(KeyError, "No key " & escape(key) & " exists!")
proc len*(node: YamlNode): int =
## If *node* is a *yMapping*, return the number of key-value pairs. If *node*
## is a *ySequence*, return the number of elements. Else, return ``0``
case node.kind
of yMapping: result = node.fields.len
of ySequence: result = node.elems.len
of yScalar: result = 0
iterator items*(node: YamlNode): YamlNode =
## Iterates over all items of a sequence. *node* must be a *ySequence*.
assert node.kind == ySequence
for item in node.elems: yield item
iterator mitems*(node: var YamlNode): YamlNode =
## Iterates over all items of a sequence. *node* must be a *ySequence*.
## Values can be modified.
assert node.kind == ySequence
for item in node.elems.mitems: yield item
iterator pairs*(node: YamlNode): tuple[key, value: YamlNode] =
## Iterates over all key-value pairs of a mapping. *node* must be a
## *yMapping*.
assert node.kind == yMapping
for key, value in node.fields: yield (key, value)
iterator mpairs*(node: var YamlNode):
tuple[key: YamlNode, value: var YamlNode] =
## Iterates over all key-value pairs of a mapping. *node* must be a
## *yMapping*. Values can be modified.
doAssert node.kind == yMapping
for key, value in node.fields.mpairs: yield (key, value)

View File

@ -124,6 +124,9 @@ template advanceTypeHint(ch: char) {.dirty.} =
ythMonth1 => ythMonthMinusNoYmd
ythMonth2 => ythMonthMinus
[ythFraction, ythSecond2] => ythAfterTimePlusMinus
of '_':
[ythInt1, ythInt2, ythInt3, ythInt4] => ythInt
[ythInt, ythDecimal] => nil
of ':':
[ythHour1, ythHour2] => ythHourColon
ythMinute2 => ythMinuteColon

View File

@ -429,21 +429,14 @@ proc endLevel(c: ParserContext, e: var YamlStreamEvent):
LevelEndResult =
result = lerOne
case c.level.kind
of fplSequence:
e = endSeqEvent()
of fplMapKey:
e = endMapEvent()
of fplSequence: e = endSeqEvent()
of fplMapKey: e = endMapEvent()
of fplMapValue, fplSinglePairValue:
e = emptyScalar(c)
c.level.kind = fplMapKey
result = lerAdditionalMapEnd
of fplUnknown:
if c.ancestry.len > 1:
e = emptyScalar(c) # don't yield scalar for empty doc
else:
result = lerNothing
of fplDocument:
e = endDocEvent()
of fplUnknown: e = emptyScalar(c)
of fplDocument: e = endDocEvent()
of fplSinglePairKey:
internalError("Unexpected level kind: " & $c.level.kind)

View File

@ -453,7 +453,6 @@ proc doPresent(s: var YamlStream, target: PresenterTarget,
case item.kind
of yamlStartDoc:
if options.style != psJson:
# TODO: tag directives
try:
case options.outputVersion
of ov1_2: target.append("%YAML 1.2" & newline)
@ -728,7 +727,7 @@ proc present*(s: var YamlStream, tagLib: TagLibrary,
doPresent(s, addr result, tagLib, options)
proc doTransform(input: Stream | string, output: PresenterTarget,
options: PresentationOptions = defaultPresentationOptions) =
options: PresentationOptions, resolveToCoreYamlTags: bool) =
var
taglib = initExtendedTagLibrary()
parser = newYamlParser(tagLib)
@ -737,30 +736,36 @@ proc doTransform(input: Stream | string, output: PresenterTarget,
if options.style == psCanonical:
var bys: YamlStream = newBufferYamlStream()
for e in events:
var event = e
case event.kind
of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq:
discard
of yamlStartMap:
if event.mapTag in [yTagQuestionMark, yTagExclamationMark]:
event.mapTag = yTagMapping
of yamlStartSeq:
if event.seqTag in [yTagQuestionMark, yTagExclamationMark]:
event.seqTag = yTagSequence
of yamlScalar:
if event.scalarTag == yTagQuestionMark:
case guessType(event.scalarContent)
of yTypeInteger: event.scalarTag = yTagInteger
of yTypeFloat, yTypeFloatInf, yTypeFloatNaN:
event.scalarTag = yTagFloat
of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean
of yTypeNull: event.scalarTag = yTagNull
of yTypeUnknown: event.scalarTag = yTagString
elif event.scalarTag == yTagExclamationMark:
event.scalarTag = yTagString
BufferYamlStream(bys).put(e)
present(bys, output, tagLib, options)
else: present(events, output, tagLib, options)
if resolveToCoreYamlTags:
var event = e
case event.kind
of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq:
discard
of yamlStartMap:
if event.mapTag in [yTagQuestionMark, yTagExclamationMark]:
event.mapTag = yTagMapping
of yamlStartSeq:
if event.seqTag in [yTagQuestionMark, yTagExclamationMark]:
event.seqTag = yTagSequence
of yamlScalar:
if event.scalarTag == yTagQuestionMark:
case guessType(event.scalarContent)
of yTypeInteger: event.scalarTag = yTagInteger
of yTypeFloat, yTypeFloatInf, yTypeFloatNaN:
event.scalarTag = yTagFloat
of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean
of yTypeNull: event.scalarTag = yTagNull
of yTypeTimestamp: event.scalarTag = yTagTimestamp
of yTypeUnknown: event.scalarTag = yTagString
elif event.scalarTag == yTagExclamationMark:
event.scalarTag = yTagString
BufferYamlStream(bys).put(event)
else: BufferYamlStream(bys).put(e)
when output is ptr[string]: output[] = present(bys, tagLib, options)
else: present(bys, output, tagLib, options)
else:
when output is ptr[string]: output[] = present(events, tagLib, options)
else: present(events, output, tagLib, options)
except YamlStreamError:
var e = getCurrentException()
while e.parent of YamlStreamError: e = e.parent
@ -769,20 +774,25 @@ proc doTransform(input: Stream | string, output: PresenterTarget,
else: internalError("Unexpected exception: " & e.parent.repr)
proc transform*(input: Stream | string, output: Stream,
options: PresentationOptions = defaultPresentationOptions)
options: PresentationOptions = defaultPresentationOptions,
resolveToCoreYamlTags: bool = false)
{.raises: [IOError, YamlParserError, YamlPresenterJsonError,
YamlPresenterOutputError].} =
## Parser ``input`` as YAML character stream and then dump it to ``output``
## while resolving non-specific tags to the ones in the YAML core tag
## library.
doTransform(input, output, options)
## library. If ``resolveToCoreYamlTags`` is ``true``, non-specific tags will
## be replaced by specific tags according to the YAML core schema.
doTransform(input, output, options, resolveToCoreYamlTags)
proc transform*(input: Stream | string,
options: PresentationOptions = defaultPresentationOptions):
options: PresentationOptions = defaultPresentationOptions,
resolveToCoreYamlTags: bool = false):
string {.raises: [IOError, YamlParserError, YamlPresenterJsonError,
YamlPresenterOutputError].} =
## Parser ``input`` as YAML character stream, resolves non-specific tags to
## the ones in the YAML core tag library, and then returns a serialized
## YAML string that represents the stream.
## YAML string that represents the stream. If ``resolveToCoreYamlTags`` is
## ``true``, non-specific tags will be replaced by specific tags according to
## the YAML core schema.
result = ""
doTransform(input, addr result, options)
doTransform(input, addr result, options, resolveToCoreYamlTags)

View File

@ -16,7 +16,7 @@
## type. Please consult the serialization guide on the NimYAML website for more
## information.
import tables, typetraits, strutils, macros, streams, times
import tables, typetraits, strutils, macros, streams, times, parseutils
import parser, taglib, presenter, stream, ../private/internal, hints
export stream
# *something* in here needs externally visible `==`(x,y: AnchorId),
@ -138,7 +138,8 @@ template constructScalarItem*(s: var YamlStream, i: untyped,
except YamlConstructionError: raise
except Exception:
var e = constructionError(s,
"Cannot construct to " & name(t) & ": " & item.scalarContent)
"Cannot construct to " & name(t) & ": " & item.scalarContent &
"; error: " & getCurrentExceptionMsg())
e.parent = getCurrentException()
raise e
@ -567,6 +568,19 @@ proc yamlTag*(T: typedesc[tuple]):
try: serializationTagLibrary.tags[uri]
except KeyError: serializationTagLibrary.registerUri(uri)
iterator recListItems(n: NimNode): NimNode =
if n.kind == nnkRecList:
for item in n.children: yield item
else: yield n
proc recListLen(n: NimNode): int {.compileTime.} =
if n.kind == nnkRecList: result = n.len
else: result = 1
proc recListNode(n: NimNode): NimNode {.compileTime.} =
if n.kind == nnkRecList: result = n[0]
else: result = n
proc fieldCount(t: typedesc): int {.compileTime.} =
result = 0
let tDesc = getType(getType(t)[1])
@ -581,13 +595,11 @@ proc fieldCount(t: typedesc): int {.compileTime.} =
for bIndex in 1..<len(child):
var recListIndex = 0
case child[bIndex].kind
of nnkOfBranch:
while child[bIndex][recListIndex].kind == nnkIntLit:
inc(recListIndex)
of nnkOfBranch: recListIndex = child[bIndex].len - 1
of nnkElse: discard
else: internalError("Unexpected child kind: " & $child[bIndex].kind)
if child[bIndex].len > recListIndex:
inc(result, child[bIndex][recListIndex].len)
inc(result, child[bIndex][recListIndex].recListLen)
macro matchMatrix(t: typedesc): untyped =
result = newNimNode(nnkBracket)
@ -681,13 +693,13 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, tIndex: int, o: typed,
recListIndex = 0
case child[bIndex].kind
of nnkOfBranch:
while recListIndex < child[bIndex].len and
child[bIndex][recListIndex].kind == nnkIntLit:
while recListIndex < child[bIndex].len - 1:
expectKind(child[bIndex][recListIndex], nnkIntLit)
curValues.add(child[bIndex][recListIndex])
inc(recListIndex)
of nnkElse: discard
else: internalError("Unexpected child kind: " & $child[bIndex].kind)
for item in child[bIndex][recListIndex].children:
for item in child[bIndex][recListIndex].recListItems:
inc(field)
discChecks.add(checkMissing(s, t, tName, item, field, matched, o,
defaultValues))
@ -738,7 +750,8 @@ macro constructFieldValue(t: typedesc, tIndex: int, stream: untyped,
case child[bIndex].kind
of nnkOfBranch:
discTest = newNimNode(nnkCurly)
while child[bIndex][recListIndex].kind == nnkIntLit:
while recListIndex < child[bIndex].len - 1:
yAssert child[bIndex][recListIndex].kind == nnkIntLit
discTest.add(child[bIndex][recListIndex])
alreadyUsedSet.add(child[bIndex][recListIndex])
inc(recListIndex)
@ -747,9 +760,8 @@ macro constructFieldValue(t: typedesc, tIndex: int, stream: untyped,
discTest = infix(discriminant, "notin", alreadyUsedSet)
else:
internalError("Unexpected child kind: " & $child[bIndex].kind)
doAssert child[bIndex][recListIndex].kind == nnkRecList
for item in child[bIndex][recListIndex].children:
for item in child[bIndex][recListIndex].recListItems:
inc(fieldIndex)
yAssert item.kind == nnkSym
var ob = newNimNode(nnkOfBranch).add(newStrLitNode($item))
@ -898,17 +910,17 @@ macro genRepresentObject(t: typedesc, value, childTagStyle: typed): typed =
case child[bIndex].kind
of nnkOfBranch:
curBranch = newNimNode(nnkOfBranch)
while child[bIndex][recListIndex].kind == nnkIntLit:
while recListIndex < child[bIndex].len - 1:
expectKind(child[bIndex][recListIndex], nnkIntLit)
curBranch.add(newCall(enumName, newLit(child[bIndex][recListIndex].intVal)))
inc(recListIndex)
of nnkElse:
curBranch = newNimNode(nnkElse)
else:
internalError("Unexpected child kind: " & $child[bIndex].kind)
doAssert child[bIndex][recListIndex].kind == nnkRecList
var curStmtList = newStmtList()
if child[bIndex][recListIndex].len > 0:
for item in child[bIndex][recListIndex].children:
if child[bIndex][recListIndex].recListLen > 0:
for item in child[bIndex][recListIndex].recListItems():
inc(fieldIndex)
let
name = $item
@ -995,16 +1007,16 @@ macro constructImplicitVariantObject(s, c, r, possibleTagIds: untyped,
yAssert recCase[i].kind == nnkOfBranch
var branch = newNimNode(nnkElifBranch)
var branchContent = newStmtList(newAssignment(discriminant, recCase[i][0]))
case recCase[i][1].len
case recCase[i][1].recListLen
of 0:
branch.add(infix(newIdentNode("yTagNull"), "in", possibleTagIds))
branchContent.add(newNimNode(nnkDiscardStmt).add(newCall("next", s)))
of 1:
let field = newDotExpr(r, newIdentNode($recCase[i][1][0]))
let field = newDotExpr(r, newIdentNode($recCase[i][1].recListNode))
branch.add(infix(
newCall("yamlTag", newCall("type", field)), "in", possibleTagIds))
branchContent.add(newCall("constructChild", s, c, field))
else: internalError("Too many children: " & $recCase[i][1].len)
else: internalError("Too many children: " & $recCase[i][1].recListlen)
branch.add(branchContent)
ifStmt.add(branch)
let raiseStmt = newNimNode(nnkRaiseStmt).add(
@ -1324,7 +1336,7 @@ proc canBeImplicit(t: typedesc): bool {.compileTime.} =
if tDesc[2][0].kind != nnkRecCase: return false
var foundEmptyBranch = false
for i in 1.. tDesc[2][0].len - 1:
case tDesc[2][0][i][1].len # branch contents
case tDesc[2][0][i][1].recListlen # branch contents
of 0:
if foundEmptyBranch: return false
else: foundEmptyBranch = true
@ -1370,8 +1382,7 @@ macro getFieldIndex(t: typedesc, field: untyped): untyped =
else:
internalError("Unexpected child kind: " &
$child[bIndex][bChildIndex].kind)
yAssert child[bIndex][bChildIndex].kind == nnkRecList
for item in child[bIndex][bChildIndex].children:
for item in child[bIndex][bChildIndex].recListItems:
inc(fieldIndex)
yAssert item.kind == nnkSym
if $item == fieldName:
@ -1391,6 +1402,21 @@ macro getFieldIndex(t: typedesc, field: untyped): untyped =
result.intVal = fieldIndex
macro markAsTransient*(t: typedesc, field: untyped): typed =
## Mark an object field as *transient*, meaning that this object field will
## not be serialized when an object instance is dumped as YAML, and also that
## the field is not expected to be given in YAML input that is loaded to an
## object instance.
##
## Example usage:
##
## .. code-block::
## type MyObject = object
## a, b: string
## c: int
## markAsTransient(MyObject, a)
## markAsTransient(MyObject, c)
##
## This does not work if the object has been marked as implicit.
let nextBitvectorIndex = transientVectors.len
result = quote do:
when compiles(`implicitVariantObjectMarker`(`t`)):
@ -1401,9 +1427,23 @@ macro markAsTransient*(t: typedesc, field: untyped): typed =
`nextBitvectorIndex`
static: transientVectors.add({})
static:
transientVectors[`transientBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`))
transientVectors[`transientBitvectorProc`(`t`)].incl(
getFieldIndex(`t`, `field`))
macro setDefaultValue*(t: typedesc, field: untyped, value: typed): typed =
## Set the default value of an object field. Fields with default values may
## be absent in YAML input when loading an instance of the object. If the
## field is absent in the YAML input, the default value is assigned to the
## field.
##
## Example usage:
##
## .. code-block::
## type MyObject = object
## a, b: string
## c: tuple[x, y: int]
## setDefaultValue(MyObject, a, "foo")
## setDefaultValue(MyObject, c, (1, 2))
let
dSym = genSym(nskVar, ":default")
nextBitvectorIndex = defaultVectors.len
@ -1424,6 +1464,16 @@ macro setDefaultValue*(t: typedesc, field: untyped, value: typed): typed =
defaultVectors[`defaultBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`))
macro ignoreInputKey*(t: typedesc, name: string{lit}): typed =
## Tell NimYAML that when loading an object of type ``t``, any mapping key
## named ``name`` shall be ignored. This makes it possible to only load
## relevant parts of a YAML input and ignoring other portions of the input.
##
## Example usage:
##
## .. code-block::
## type MyObject = object
## a, b: string
## ignoreInputKey(MyObject, "c")
let nextIgnoredKeyList = ignoredKeyLists.len
result = quote do:
when not compiles(`ignoredKeyListProc`(`t`)):