diff --git a/.gitignore b/.gitignore index 7629330..30eef3e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ test/tdom test/tserialization test/tjson test/tparser -test/yamlTestSuite test/tquickstart test/*.exe test/*.pdb @@ -22,5 +21,5 @@ docout doc/rstPreproc doc/tmp.rst doc/**/code -yaml-dev-kit +test/yaml-test-suite nimsuggest.log diff --git a/CHANGELOG.md b/CHANGELOG.md index 290d445..020a7f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +### 0.9.0 + +Features: + + * Better DOM API: + - yMapping is now a Table + - field names have changed to imitate those of Nim's json API + - Better getter and setter procs + * Added ability to resolve non-specific tags in presenter.transform + +Bugfixes: + + * Fixed parsing floating point literals (#30) + * Fixed a bug with variant records (#31) + * Empty documents now always contain an empty scalar + ### 0.8.0 Features: diff --git a/doc/testing.rst b/doc/testing.rst index a6a3b01..4aadb55 100644 --- a/doc/testing.rst +++ b/doc/testing.rst @@ -74,7 +74,7 @@ updated as you type. var params = "style=" + encodeURIComponent(document.querySelector( "input[name=style]:checked").value) + "&input=" + encodeURIComponent( document.getElementById("yaml-input").value); - r.open("POST", "http://flyx.org:5000", true); + r.open("POST", "https://nimyaml.org/webservice/", true); r.setRequestHeader("Content-type", "application/x-www-form-urlencoded"); r.onreadystatechange = function() { if (r.readyState == 4) { diff --git a/private/lex.nim b/private/lex.nim index 048e7e1..47e6fec 100644 --- a/private/lex.nim +++ b/private/lex.nim @@ -897,6 +897,7 @@ proc blockScalar[T](lex: YamlLexer): bool = break else: lex.blockScalarIndent += lex.indentation + lex.indentation = 0 if lex.c notin {'.', '-'} or lex.indentation == 0: if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer else: diff --git a/server/server.nim b/server/server.nim index 33edd10..7927869 100644 --- a/server/server.nim +++ b/server/server.nim @@ -47,7 +47,7 @@ routes: var output = newStringStream() highlighted = "" - transform(newStringStream(@"input"), output, defineOptions(style)) + transform(newStringStream(@"input"), output, defineOptions(style), true) # syntax highlighting (stolen and modified from stlib's rstgen) var g: GeneralTokenizer diff --git a/test/tdom.nim b/test/tdom.nim index 2aa5c33..6b4606a 100644 --- a/test/tdom.nim +++ b/test/tdom.nim @@ -5,7 +5,7 @@ # distribution, for details about the copyright. import "../yaml" -import unittest, commonTestUtils, streams +import unittest, commonTestUtils, streams, tables suite "DOM": test "Composing simple Scalar": @@ -25,13 +25,13 @@ suite "DOM": result = loadDOM(input) assert result.root.kind == ySequence assert result.root.tag == "?" - assert result.root.children.len == 2 - assert result.root.children[0].kind == yScalar - assert result.root.children[0].tag == "tag:yaml.org,2002:str" - assert result.root.children[0].content == "a" - assert result.root.children[1].kind == yScalar - assert result.root.children[1].tag == "tag:yaml.org,2002:bool" - assert result.root.children[1].content == "no" + assert result.root.len == 2 + assert result.root[0].kind == yScalar + assert result.root[0].tag == "tag:yaml.org,2002:str" + assert result.root[0].content == "a" + assert result.root[1].kind == yScalar + assert result.root[1].tag == "tag:yaml.org,2002:bool" + assert result.root[1].content == "no" test "Serializing sequence": let input = initYamlDoc(newYamlNode([ newYamlNode("a", "tag:yaml.org,2002:str"), @@ -46,12 +46,13 @@ suite "DOM": result = loadDOM(input) assert result.root.kind == yMapping assert result.root.tag == "tag:yaml.org,2002:map" - assert result.root.pairs.len == 1 - assert result.root.pairs[0].key.kind == yScalar - assert result.root.pairs[0].key.tag == "!foo" - assert result.root.pairs[0].key.content == "bar" - assert result.root.pairs[0].value.kind == ySequence - assert result.root.pairs[0].value.children.len == 2 + assert result.root.fields.len == 1 + for key, value in result.root.fields.pairs: + assert key.kind == yScalar + assert key.tag == "!foo" + assert key.content == "bar" + assert value.kind == ySequence + assert value.len == 2 test "Serializing mapping": let input = initYamlDoc(newYamlNode([ (key: newYamlNode("bar"), value: newYamlNode([newYamlNode("a"), @@ -65,13 +66,13 @@ suite "DOM": input = newStringStream("- &a foo\n- &b bar\n- *a\n- *b") result = loadDOM(input) assert result.root.kind == ySequence - assert result.root.children.len == 4 - assert result.root.children[0].kind == yScalar - assert result.root.children[0].content == "foo" - assert result.root.children[1].kind == yScalar - assert result.root.children[1].content == "bar" - assert result.root.children[0] == result.root.children[2] - assert result.root.children[1] == result.root.children[3] + assert result.root.len == 4 + assert result.root[0].kind == yScalar + assert result.root[0].content == "foo" + assert result.root[1].kind == yScalar + assert result.root[1].content == "bar" + assert cast[pointer](result.root[0]) == cast[pointer](result.root[2]) + assert cast[pointer](result.root[1]) == cast[pointer](result.root[3]) test "Serializing with anchors": let a = newYamlNode("a") diff --git a/test/testEventParser.nim b/test/testEventParser.nim index f99e4a6..4db6b25 100644 --- a/test/testEventParser.nim +++ b/test/testEventParser.nim @@ -5,12 +5,13 @@ # distribution, for details about the copyright. import "../yaml" -import lexbase, streams, tables +import lexbase, streams, tables, strutils type LexerToken = enum plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, minusSeq, - eqVal, eqAli, chevTag, andAnchor, quotContent, colonContent, noToken + eqVal, eqAli, chevTag, andAnchor, starAnchor, quotContent, colonContent, + explDirEnd, explDocEnd, noToken StreamPos = enum beforeStream, inStream, afterStream @@ -29,7 +30,7 @@ proc nextToken(lex: var EventLexer): LexerToken = else: break if lex.buf[lex.bufpos] == EndOfFile: return noToken case lex.buf[lex.bufpos] - of ':', '"': + of ':', '"', '\'', '|', '>': let t = if lex.buf[lex.bufpos] == ':': colonContent else: quotContent lex.content = "" lex.bufpos.inc() @@ -73,6 +74,13 @@ proc nextToken(lex: var EventLexer): LexerToken = lex.content.add(lex.buf[lex.bufpos]) lex.bufpos.inc() result = andAnchor + of '*': + lex.content = "" + lex.bufpos.inc() + while lex.buf[lex.bufpos] notin {' ', '\t', '\r', '\l', EndOfFile}: + lex.content.add(lex.buf[lex.bufpos]) + lex.bufpos.inc() + result = starAnchor else: lex.content = "" while lex.buf[lex.bufpos] notin {' ', '\t', '\r', '\l', EndOfFile}: @@ -89,6 +97,8 @@ proc nextToken(lex: var EventLexer): LexerToken = of "-SEQ": result = minusSeq of "=VAL": result = eqVal of "=ALI": result = eqAli + of "---": result = explDirEnd + of "...": result = explDocEnd else: raise newException(EventStreamError, "Invalid token: " & lex.content) template assertInStream() {.dirty.} = @@ -210,12 +220,19 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream = if curAnchor() != yAnchorNone: raise newException(EventStreamError, "Duplicate anchor in " & $curEvent.kind) - if curEvent.kind == yamlAlias: - curEvent.aliasTarget = anchors[lex.content] + anchors[lex.content] = nextAnchorId + setCurAnchor(nextAnchorId) + nextAnchorId = (AnchorId)(((int)nextAnchorId) + 1) + of starAnchor: + assertInEvent("alias") + if curEvent.kind != yamlAlias: + raise newException(EventStreamError, "Unexpected alias: " & + escape(lex.content)) + elif curEvent.aliasTarget != yAnchorNone: + raise newException(EventStreamError, "Duplicate alias target: " & + escape(lex.content)) else: - anchors[lex.content] = nextAnchorId - setCurAnchor(nextAnchorId) - nextAnchorId = (AnchorId)(((int)nextAnchorId) + 1) + curEvent.aliasTarget = anchors[lex.content] of quotContent: assertInEvent("scalar content") if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark) @@ -229,5 +246,14 @@ proc parseEventStream*(input: Stream, tagLib: TagLibrary): YamlStream = if curEvent.kind != yamlScalar: raise newException(EventStreamError, "scalar content in non-scalar tag") + of explDirEnd: + assertInEvent("explicit directives end") + if curEvent.kind != yamlStartDoc: + raise newException(EventStreamError, + "Unexpected explicit directives end") + of explDocEnd: + if curEvent.kind != yamlEndDoc: + raise newException(EventStreamError, + "Unexpected explicit document end") of noToken: discard result = initYamlStream(backend) \ No newline at end of file diff --git a/test/tparser.nim b/test/tparser.nim index 9ace20b..7c44be5 100644 --- a/test/tparser.nim +++ b/test/tparser.nim @@ -8,19 +8,21 @@ import os, osproc, terminal, strutils, streams, macros, unittest import testEventParser, commonTestUtils import "../yaml" -const devKitFolder = "yaml-dev-kit" +const + testSuiteFolder = "yaml-test-suite" + testSuiteUrl = "https://github.com/yaml/yaml-test-suite.git" proc echoError(msg: string) = styledWriteLine(stdout, fgRed, "[error] ", fgWhite, msg, resetStyle) -proc ensureDevKitCloneCorrect(pwd: string) {.compileTime.} = - let absolutePath = pwd / devKitFolder +proc ensureTestSuiteCloneCorrect(pwd: string) {.compileTime.} = + let absolutePath = pwd / testSuiteFolder if dirExists(absolutePath): var isCorrectClone = true if dirExists(absolutePath / ".git"): let remoteUrl = staticExec("cd \"" & absolutePath & "\" && git remote get-url origin") - if remoteUrl != "https://github.com/ingydotnet/yaml-dev-kit.git": + if remoteUrl != testSuiteUrl: isCorrectClone = false let branches = staticExec("cd \"" & absolutePath & "\" && git branch") if "* data" notin branches.splitLines(): @@ -28,25 +30,25 @@ proc ensureDevKitCloneCorrect(pwd: string) {.compileTime.} = if isCorrectClone: let updateOutput = staticExec("cd \"" & absolutePath & "\" && git pull") #if uError != 0: - # echo "could not update yaml-dev-kit! please fix this problem and compile again." + # echo "could not update yaml-test-suite! please fix this problem and compile again." # echo "output:\n" # echo "$ git pull" # echo updateOutput # quit 1 else: - echo devKitFolder, " exists, but is not in expected state. Make sure it is a git repo," - echo "cloned from https://github.com/ingydotnet/yaml-dev-kit.git, and the data branch" - echo "is active. Alternatively, delete the folder " & devKitFolder & '.' + echo testSuiteFolder, " exists, but is not in expected state. Make sure it is a git repo," + echo "cloned from ", testSuiteUrl, ", and the data branch" + echo "is active. Alternatively, delete the folder " & testSuiteFolder & '.' quit 1 else: let cloneOutput = staticExec("cd \"" & pwd & - "\" && git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data") + "\" && git clone " & testSuiteUrl & " -b data") #if cError != 0: if not(dirExists(absolutePath)) or not(dirExists(absolutePath / ".git")) or not(dirExists(absolutePath / "229Q")): - echo "could not clone https://github.com/ingydotnet/yaml-dev-kit.git. Make sure" + echo "could not clone ", testSuiteUrl, ". Make sure" echo "you are connected to the internet and your proxy settings are correct. output:\n" - echo "$ git clone https://github.com/ingydotnet/yaml-dev-kit.git" + echo "$ git clone ", testSuiteUrl, " -b data" echo cloneOutput quit 1 @@ -95,9 +97,9 @@ proc parserTest(path: string): bool = macro genTests(): untyped = let pwd = staticExec("pwd") - absolutePath = '"' & (pwd / devKitFolder) & '"' + absolutePath = '"' & (pwd / testSuiteFolder) & '"' echo "[tparser] Generating tests from " & absolutePath - ensureDevKitCloneCorrect(pwd) + ensureTestSuiteCloneCorrect(pwd) result = newStmtList() # walkDir for some crude reason does not work with travis build let dirItems = staticExec("ls -1d " & absolutePath / "*") @@ -108,6 +110,6 @@ macro genTests(): untyped = newLit(strip(title) & " [" & dirPath[^4..^1] & ']'), newCall("doAssert", newCall("parserTest", newLit(dirPath))))) - result = newCall("suite", newLit("Parser Tests (from yaml-dev-kit)"), result) + result = newCall("suite", newLit("Parser Tests (from yaml-test-suite)"), result) genTests() \ No newline at end of file diff --git a/test/tserialization.nim b/test/tserialization.nim index 1261132..70a6066 100644 --- a/test/tserialization.nim +++ b/test/tserialization.nim @@ -5,7 +5,7 @@ # distribution, for details about the copyright. import "../yaml" -import unittest, strutils, streams, tables, times +import unittest, strutils, streams, tables, times, math type MyTuple = tuple @@ -35,8 +35,7 @@ type case kind: AnimalKind of akCat: purringIntensity: int - of akDog: - barkometer: int + of akDog: barkometer: int DumbEnum = enum deA, deB, deC, deD @@ -197,6 +196,15 @@ suite "Serialization": load(input, result) assert(result == 14) + test "Load floats": + let input = "[6.8523015e+5, 685.230_15e+03, 685_230.15, -.inf, .NaN]" + var result: seq[float] + load(input, result) + for i in 0..2: + assert result[i] == 6.8523015e+5 + assert result[3] == NegInf + assert classify(result[4]) == fcNan + test "Load nil string": let input = newStringStream("! \"\"") var result: string diff --git a/yaml.nimble b/yaml.nimble index ceb5b6f..47d1f6b 100644 --- a/yaml.nimble +++ b/yaml.nimble @@ -1,6 +1,6 @@ # Package -version = "0.8.0" +version = "0.9.0" author = "Felix Krause" description = "YAML 1.2 implementation for Nim" license = "MIT" diff --git a/yaml/dom.nim b/yaml/dom.nim index f4b3ee6..2b5a563 100644 --- a/yaml/dom.nim +++ b/yaml/dom.nim @@ -12,11 +12,14 @@ ## structure. It can also dump the structure back to YAML. Formally, it ## represents the *Representation Graph* as defined in the YAML specification. ## -## The main interface of this API are ``loadDOM`` and ``dumpDOM``. The other +## The main interface of this API are ``loadDom`` and ``dumpDom``. The other ## exposed procs are low-level and useful if you want to load or generate parts ## of a ``YamlStream``. +## +## The ``YamlNode`` objects in the DOM can be used similarly to the ``JsonNode`` +## objects of Nim's `json module `_. -import tables, streams +import tables, streams, hashes, sets, strutils import stream, taglib, serialization, ../private/internal, parser, presenter @@ -31,23 +34,98 @@ type tag*: string case kind*: YamlNodeKind of yScalar: content*: string - of ySequence: children*: seq[YamlNode] - of yMapping: pairs*: seq[tuple[key, value: YamlNode]] + of ySequence: elems*: seq[YamlNode] + of yMapping: fields*: TableRef[YamlNode, YamlNode] + # compiler does not like Table[YamlNode, YamlNode] YamlDocument* = object ## Represents a YAML document. root*: YamlNode +proc hash*(o: YamlNode): Hash = + result = o.tag.hash + case o.kind + of yScalar: result = result !& o.content.hash + of yMapping: + for key, value in o.fields.pairs: + result = result !& key.hash !& value.hash + of ySequence: + for item in o.elems: + result = result !& item.hash + result = !$result + +proc eqImpl(x, y: YamlNode, alreadyVisited: var HashSet[pointer]): bool = + template compare(a, b: YamlNode) {.dirty.} = + if cast[pointer](a) != cast[pointer](b): + if cast[pointer](a) in alreadyVisited and + cast[pointer](b) in alreadyVisited: + # prevent infinite loop! + return false + elif a != b: return false + + if x.kind != y.kind or x.tag != y.tag: return false + alreadyVisited.incl(cast[pointer](x)) + alreadyVisited.incl(cast[pointer](y)) + case x.kind + of yScalar: result = x.content == y.content + of ySequence: + if x.elems.len != y.elems.len: return false + for i in 0.. " + case n.kind + of yScalar: result.add(escape(n.content)) + of ySequence: + result.add('[') + for item in n.elems: + result.add($item) + result.add(", ") + result.setLen(result.len - 1) + result[^1] = ']' + of yMapping: + result.add('{') + for key, value in n.fields.pairs: + result.add($key) + result.add(": ") + result.add($value) + result.add(", ") + result.setLen(result.len - 1) + result[^1] = '}' + proc newYamlNode*(content: string, tag: string = "?"): YamlNode = YamlNode(kind: yScalar, content: content, tag: tag) -proc newYamlNode*(children: openarray[YamlNode], tag: string = "?"): +proc newYamlNode*(elems: openarray[YamlNode], tag: string = "?"): YamlNode = - YamlNode(kind: ySequence, children: @children, tag: tag) + YamlNode(kind: ySequence, elems: @elems, tag: tag) -proc newYamlNode*(pairs: openarray[tuple[key, value: YamlNode]], +proc newYamlNode*(fields: openarray[(YamlNode, YamlNode)], tag: string = "?"): YamlNode = - YamlNode(kind: yMapping, pairs: @pairs, tag: tag) + YamlNode(kind: yMapping, fields: newTable(fields), tag: tag) proc initYamlDoc*(root: YamlNode): YamlDocument = result.root = root @@ -62,12 +140,14 @@ proc composeNode(s: var YamlStream, tagLib: TagLibrary, of yamlStartMap: result.tag = tagLib.uri(start.mapTag) result.kind = yMapping - result.pairs = newSeq[tuple[key, value: YamlNode]]() + result.fields = newTable[YamlNode, YamlNode]() while s.peek().kind != yamlEndMap: let key = composeNode(s, tagLib, c) value = composeNode(s, tagLib, c) - result.pairs.add((key: key, value: value)) + if result.fields.hasKeyOrPut(key, value): + raise newException(YamlConstructionError, + "Duplicate key: " & $key) discard s.next() if start.mapAnchor != yAnchorNone: yAssert(not c.refs.hasKey(start.mapAnchor)) @@ -75,9 +155,9 @@ proc composeNode(s: var YamlStream, tagLib: TagLibrary, of yamlStartSeq: result.tag = tagLib.uri(start.seqTag) result.kind = ySequence - result.children = newSeq[YamlNode]() + result.elems = newSeq[YamlNode]() while s.peek().kind != yamlEndSeq: - result.children.add(composeNode(s, tagLib, c)) + result.elems.add(composeNode(s, tagLib, c)) if start.seqAnchor != yAnchorNone: yAssert(not c.refs.hasKey(start.seqAnchor)) c.refs[start.seqAnchor] = cast[pointer](result) @@ -106,7 +186,7 @@ proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument n = s.next() yAssert n.kind == yamlEndDoc -proc loadDOM*(s: Stream | string): YamlDocument +proc loadDom*(s: Stream | string): YamlDocument {.raises: [IOError, YamlParserError, YamlConstructionError].} = var tagLib = initExtendedTagLibrary() @@ -148,14 +228,14 @@ proc serializeNode(n: YamlNode, c: SerializationContext, a: AnchorStyle, of yScalar: c.put(scalarEvent(n.content, tagId, anchor)) of ySequence: c.put(startSeqEvent(tagId, anchor)) - for item in n.children: + for item in n.elems: serializeNode(item, c, a, tagLib) c.put(endSeqEvent()) of yMapping: c.put(startMapEvent(tagId, anchor)) - for i in n.pairs: - serializeNode(i.key, c, a, tagLib) - serializeNode(i.value, c, a, tagLib) + for key, value in n.fields.pairs: + serializeNode(key, c, a, tagLib) + serializeNode(value, c, a, tagLib) c.put(endMapEvent()) template processAnchoredEvent(target: untyped, c: SerializationContext): typed = @@ -182,7 +262,7 @@ proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy): else: discard result = bys -proc dumpDOM*(doc: YamlDocument, target: Stream, +proc dumpDom*(doc: YamlDocument, target: Stream, anchorStyle: AnchorStyle = asTidy, options: PresentationOptions = defaultPresentationOptions) {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, @@ -193,3 +273,70 @@ proc dumpDOM*(doc: YamlDocument, target: Stream, events = serialize(doc, tagLib, if options.style == psJson: asNone else: anchorStyle) present(events, target, tagLib, options) + +proc `[]`*(node: YamlNode, i: int): YamlNode = + ## Get the node at index *i* from a sequence. *node* must be a *ySequence*. + assert node.kind == ySequence + node.elems[i] + +proc `[]=`*(node: var YamlNode, i: int, val: YamlNode) = + ## Set the node at index *i* of a sequence. *node* must be a *ySequence*. + assert node.kind == ySequence + node.elems[i] = val + +proc `[]`*(node: YamlNode, key: YamlNode): YamlNode = + ## Get the value for a key in a mapping. *node* must be a *yMapping*. + assert node.kind == yMapping + node.fields[key] + +proc `[]=`*(node: YamlNode, key: YamlNode, value: YamlNode) = + ## Set the value for a key in a mapping. *node* must be a *yMapping*. + node.fields[key] = value + +proc `[]`*(node: YamlNode, key: string): YamlNode = + ## Get the value for a string key in a mapping. *node* must be a *yMapping*. + ## This searches for a scalar key with content *key* and either no explicit + ## tag or the explicit tag ``!!str``. + assert node.kind == yMapping + var keyNode = YamlNode(kind: yScalar, tag: "!", content: key) + result = node.fields.getOrDefault(keyNode) + if isNil(result): + keyNode.tag = "?" + result = node.fields.getOrDefault(keyNode) + if isNil(result): + keyNode.tag = nimTag(yamlTagRepositoryPrefix & "str") + result = node.fields.getOrDefault(keyNode) + if isNil(result): + raise newException(KeyError, "No key " & escape(key) & " exists!") + +proc len*(node: YamlNode): int = + ## If *node* is a *yMapping*, return the number of key-value pairs. If *node* + ## is a *ySequence*, return the number of elements. Else, return ``0`` + case node.kind + of yMapping: result = node.fields.len + of ySequence: result = node.elems.len + of yScalar: result = 0 + +iterator items*(node: YamlNode): YamlNode = + ## Iterates over all items of a sequence. *node* must be a *ySequence*. + assert node.kind == ySequence + for item in node.elems: yield item + +iterator mitems*(node: var YamlNode): YamlNode = + ## Iterates over all items of a sequence. *node* must be a *ySequence*. + ## Values can be modified. + assert node.kind == ySequence + for item in node.elems.mitems: yield item + +iterator pairs*(node: YamlNode): tuple[key, value: YamlNode] = + ## Iterates over all key-value pairs of a mapping. *node* must be a + ## *yMapping*. + assert node.kind == yMapping + for key, value in node.fields: yield (key, value) + +iterator mpairs*(node: var YamlNode): + tuple[key: YamlNode, value: var YamlNode] = + ## Iterates over all key-value pairs of a mapping. *node* must be a + ## *yMapping*. Values can be modified. + doAssert node.kind == yMapping + for key, value in node.fields.mpairs: yield (key, value) \ No newline at end of file diff --git a/yaml/hints.nim b/yaml/hints.nim index b3b6e4a..ba7737a 100644 --- a/yaml/hints.nim +++ b/yaml/hints.nim @@ -124,6 +124,9 @@ template advanceTypeHint(ch: char) {.dirty.} = ythMonth1 => ythMonthMinusNoYmd ythMonth2 => ythMonthMinus [ythFraction, ythSecond2] => ythAfterTimePlusMinus + of '_': + [ythInt1, ythInt2, ythInt3, ythInt4] => ythInt + [ythInt, ythDecimal] => nil of ':': [ythHour1, ythHour2] => ythHourColon ythMinute2 => ythMinuteColon diff --git a/yaml/parser.nim b/yaml/parser.nim index 200cf28..96c9f83 100644 --- a/yaml/parser.nim +++ b/yaml/parser.nim @@ -429,21 +429,14 @@ proc endLevel(c: ParserContext, e: var YamlStreamEvent): LevelEndResult = result = lerOne case c.level.kind - of fplSequence: - e = endSeqEvent() - of fplMapKey: - e = endMapEvent() + of fplSequence: e = endSeqEvent() + of fplMapKey: e = endMapEvent() of fplMapValue, fplSinglePairValue: e = emptyScalar(c) c.level.kind = fplMapKey result = lerAdditionalMapEnd - of fplUnknown: - if c.ancestry.len > 1: - e = emptyScalar(c) # don't yield scalar for empty doc - else: - result = lerNothing - of fplDocument: - e = endDocEvent() + of fplUnknown: e = emptyScalar(c) + of fplDocument: e = endDocEvent() of fplSinglePairKey: internalError("Unexpected level kind: " & $c.level.kind) diff --git a/yaml/presenter.nim b/yaml/presenter.nim index 9dd1151..ad59f59 100644 --- a/yaml/presenter.nim +++ b/yaml/presenter.nim @@ -453,7 +453,6 @@ proc doPresent(s: var YamlStream, target: PresenterTarget, case item.kind of yamlStartDoc: if options.style != psJson: - # TODO: tag directives try: case options.outputVersion of ov1_2: target.append("%YAML 1.2" & newline) @@ -728,7 +727,7 @@ proc present*(s: var YamlStream, tagLib: TagLibrary, doPresent(s, addr result, tagLib, options) proc doTransform(input: Stream | string, output: PresenterTarget, - options: PresentationOptions = defaultPresentationOptions) = + options: PresentationOptions, resolveToCoreYamlTags: bool) = var taglib = initExtendedTagLibrary() parser = newYamlParser(tagLib) @@ -737,30 +736,36 @@ proc doTransform(input: Stream | string, output: PresenterTarget, if options.style == psCanonical: var bys: YamlStream = newBufferYamlStream() for e in events: - var event = e - case event.kind - of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq: - discard - of yamlStartMap: - if event.mapTag in [yTagQuestionMark, yTagExclamationMark]: - event.mapTag = yTagMapping - of yamlStartSeq: - if event.seqTag in [yTagQuestionMark, yTagExclamationMark]: - event.seqTag = yTagSequence - of yamlScalar: - if event.scalarTag == yTagQuestionMark: - case guessType(event.scalarContent) - of yTypeInteger: event.scalarTag = yTagInteger - of yTypeFloat, yTypeFloatInf, yTypeFloatNaN: - event.scalarTag = yTagFloat - of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean - of yTypeNull: event.scalarTag = yTagNull - of yTypeUnknown: event.scalarTag = yTagString - elif event.scalarTag == yTagExclamationMark: - event.scalarTag = yTagString - BufferYamlStream(bys).put(e) - present(bys, output, tagLib, options) - else: present(events, output, tagLib, options) + if resolveToCoreYamlTags: + var event = e + case event.kind + of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq: + discard + of yamlStartMap: + if event.mapTag in [yTagQuestionMark, yTagExclamationMark]: + event.mapTag = yTagMapping + of yamlStartSeq: + if event.seqTag in [yTagQuestionMark, yTagExclamationMark]: + event.seqTag = yTagSequence + of yamlScalar: + if event.scalarTag == yTagQuestionMark: + case guessType(event.scalarContent) + of yTypeInteger: event.scalarTag = yTagInteger + of yTypeFloat, yTypeFloatInf, yTypeFloatNaN: + event.scalarTag = yTagFloat + of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean + of yTypeNull: event.scalarTag = yTagNull + of yTypeTimestamp: event.scalarTag = yTagTimestamp + of yTypeUnknown: event.scalarTag = yTagString + elif event.scalarTag == yTagExclamationMark: + event.scalarTag = yTagString + BufferYamlStream(bys).put(event) + else: BufferYamlStream(bys).put(e) + when output is ptr[string]: output[] = present(bys, tagLib, options) + else: present(bys, output, tagLib, options) + else: + when output is ptr[string]: output[] = present(events, tagLib, options) + else: present(events, output, tagLib, options) except YamlStreamError: var e = getCurrentException() while e.parent of YamlStreamError: e = e.parent @@ -769,20 +774,25 @@ proc doTransform(input: Stream | string, output: PresenterTarget, else: internalError("Unexpected exception: " & e.parent.repr) proc transform*(input: Stream | string, output: Stream, - options: PresentationOptions = defaultPresentationOptions) + options: PresentationOptions = defaultPresentationOptions, + resolveToCoreYamlTags: bool = false) {.raises: [IOError, YamlParserError, YamlPresenterJsonError, YamlPresenterOutputError].} = ## Parser ``input`` as YAML character stream and then dump it to ``output`` ## while resolving non-specific tags to the ones in the YAML core tag - ## library. - doTransform(input, output, options) + ## library. If ``resolveToCoreYamlTags`` is ``true``, non-specific tags will + ## be replaced by specific tags according to the YAML core schema. + doTransform(input, output, options, resolveToCoreYamlTags) proc transform*(input: Stream | string, - options: PresentationOptions = defaultPresentationOptions): + options: PresentationOptions = defaultPresentationOptions, + resolveToCoreYamlTags: bool = false): string {.raises: [IOError, YamlParserError, YamlPresenterJsonError, YamlPresenterOutputError].} = ## Parser ``input`` as YAML character stream, resolves non-specific tags to ## the ones in the YAML core tag library, and then returns a serialized - ## YAML string that represents the stream. + ## YAML string that represents the stream. If ``resolveToCoreYamlTags`` is + ## ``true``, non-specific tags will be replaced by specific tags according to + ## the YAML core schema. result = "" - doTransform(input, addr result, options) + doTransform(input, addr result, options, resolveToCoreYamlTags) diff --git a/yaml/serialization.nim b/yaml/serialization.nim index d1365d9..03944c3 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -16,7 +16,7 @@ ## type. Please consult the serialization guide on the NimYAML website for more ## information. -import tables, typetraits, strutils, macros, streams, times +import tables, typetraits, strutils, macros, streams, times, parseutils import parser, taglib, presenter, stream, ../private/internal, hints export stream # *something* in here needs externally visible `==`(x,y: AnchorId), @@ -138,7 +138,8 @@ template constructScalarItem*(s: var YamlStream, i: untyped, except YamlConstructionError: raise except Exception: var e = constructionError(s, - "Cannot construct to " & name(t) & ": " & item.scalarContent) + "Cannot construct to " & name(t) & ": " & item.scalarContent & + "; error: " & getCurrentExceptionMsg()) e.parent = getCurrentException() raise e @@ -567,6 +568,19 @@ proc yamlTag*(T: typedesc[tuple]): try: serializationTagLibrary.tags[uri] except KeyError: serializationTagLibrary.registerUri(uri) +iterator recListItems(n: NimNode): NimNode = + if n.kind == nnkRecList: + for item in n.children: yield item + else: yield n + +proc recListLen(n: NimNode): int {.compileTime.} = + if n.kind == nnkRecList: result = n.len + else: result = 1 + +proc recListNode(n: NimNode): NimNode {.compileTime.} = + if n.kind == nnkRecList: result = n[0] + else: result = n + proc fieldCount(t: typedesc): int {.compileTime.} = result = 0 let tDesc = getType(getType(t)[1]) @@ -581,13 +595,11 @@ proc fieldCount(t: typedesc): int {.compileTime.} = for bIndex in 1.. recListIndex: - inc(result, child[bIndex][recListIndex].len) + inc(result, child[bIndex][recListIndex].recListLen) macro matchMatrix(t: typedesc): untyped = result = newNimNode(nnkBracket) @@ -681,13 +693,13 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, tIndex: int, o: typed, recListIndex = 0 case child[bIndex].kind of nnkOfBranch: - while recListIndex < child[bIndex].len and - child[bIndex][recListIndex].kind == nnkIntLit: + while recListIndex < child[bIndex].len - 1: + expectKind(child[bIndex][recListIndex], nnkIntLit) curValues.add(child[bIndex][recListIndex]) inc(recListIndex) of nnkElse: discard else: internalError("Unexpected child kind: " & $child[bIndex].kind) - for item in child[bIndex][recListIndex].children: + for item in child[bIndex][recListIndex].recListItems: inc(field) discChecks.add(checkMissing(s, t, tName, item, field, matched, o, defaultValues)) @@ -738,7 +750,8 @@ macro constructFieldValue(t: typedesc, tIndex: int, stream: untyped, case child[bIndex].kind of nnkOfBranch: discTest = newNimNode(nnkCurly) - while child[bIndex][recListIndex].kind == nnkIntLit: + while recListIndex < child[bIndex].len - 1: + yAssert child[bIndex][recListIndex].kind == nnkIntLit discTest.add(child[bIndex][recListIndex]) alreadyUsedSet.add(child[bIndex][recListIndex]) inc(recListIndex) @@ -747,9 +760,8 @@ macro constructFieldValue(t: typedesc, tIndex: int, stream: untyped, discTest = infix(discriminant, "notin", alreadyUsedSet) else: internalError("Unexpected child kind: " & $child[bIndex].kind) - doAssert child[bIndex][recListIndex].kind == nnkRecList - for item in child[bIndex][recListIndex].children: + for item in child[bIndex][recListIndex].recListItems: inc(fieldIndex) yAssert item.kind == nnkSym var ob = newNimNode(nnkOfBranch).add(newStrLitNode($item)) @@ -898,17 +910,17 @@ macro genRepresentObject(t: typedesc, value, childTagStyle: typed): typed = case child[bIndex].kind of nnkOfBranch: curBranch = newNimNode(nnkOfBranch) - while child[bIndex][recListIndex].kind == nnkIntLit: + while recListIndex < child[bIndex].len - 1: + expectKind(child[bIndex][recListIndex], nnkIntLit) curBranch.add(newCall(enumName, newLit(child[bIndex][recListIndex].intVal))) inc(recListIndex) of nnkElse: curBranch = newNimNode(nnkElse) else: internalError("Unexpected child kind: " & $child[bIndex].kind) - doAssert child[bIndex][recListIndex].kind == nnkRecList var curStmtList = newStmtList() - if child[bIndex][recListIndex].len > 0: - for item in child[bIndex][recListIndex].children: + if child[bIndex][recListIndex].recListLen > 0: + for item in child[bIndex][recListIndex].recListItems(): inc(fieldIndex) let name = $item @@ -995,16 +1007,16 @@ macro constructImplicitVariantObject(s, c, r, possibleTagIds: untyped, yAssert recCase[i].kind == nnkOfBranch var branch = newNimNode(nnkElifBranch) var branchContent = newStmtList(newAssignment(discriminant, recCase[i][0])) - case recCase[i][1].len + case recCase[i][1].recListLen of 0: branch.add(infix(newIdentNode("yTagNull"), "in", possibleTagIds)) branchContent.add(newNimNode(nnkDiscardStmt).add(newCall("next", s))) of 1: - let field = newDotExpr(r, newIdentNode($recCase[i][1][0])) + let field = newDotExpr(r, newIdentNode($recCase[i][1].recListNode)) branch.add(infix( newCall("yamlTag", newCall("type", field)), "in", possibleTagIds)) branchContent.add(newCall("constructChild", s, c, field)) - else: internalError("Too many children: " & $recCase[i][1].len) + else: internalError("Too many children: " & $recCase[i][1].recListlen) branch.add(branchContent) ifStmt.add(branch) let raiseStmt = newNimNode(nnkRaiseStmt).add( @@ -1324,7 +1336,7 @@ proc canBeImplicit(t: typedesc): bool {.compileTime.} = if tDesc[2][0].kind != nnkRecCase: return false var foundEmptyBranch = false for i in 1.. tDesc[2][0].len - 1: - case tDesc[2][0][i][1].len # branch contents + case tDesc[2][0][i][1].recListlen # branch contents of 0: if foundEmptyBranch: return false else: foundEmptyBranch = true @@ -1370,8 +1382,7 @@ macro getFieldIndex(t: typedesc, field: untyped): untyped = else: internalError("Unexpected child kind: " & $child[bIndex][bChildIndex].kind) - yAssert child[bIndex][bChildIndex].kind == nnkRecList - for item in child[bIndex][bChildIndex].children: + for item in child[bIndex][bChildIndex].recListItems: inc(fieldIndex) yAssert item.kind == nnkSym if $item == fieldName: @@ -1391,6 +1402,21 @@ macro getFieldIndex(t: typedesc, field: untyped): untyped = result.intVal = fieldIndex macro markAsTransient*(t: typedesc, field: untyped): typed = + ## Mark an object field as *transient*, meaning that this object field will + ## not be serialized when an object instance is dumped as YAML, and also that + ## the field is not expected to be given in YAML input that is loaded to an + ## object instance. + ## + ## Example usage: + ## + ## .. code-block:: + ## type MyObject = object + ## a, b: string + ## c: int + ## markAsTransient(MyObject, a) + ## markAsTransient(MyObject, c) + ## + ## This does not work if the object has been marked as implicit. let nextBitvectorIndex = transientVectors.len result = quote do: when compiles(`implicitVariantObjectMarker`(`t`)): @@ -1401,9 +1427,23 @@ macro markAsTransient*(t: typedesc, field: untyped): typed = `nextBitvectorIndex` static: transientVectors.add({}) static: - transientVectors[`transientBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`)) + transientVectors[`transientBitvectorProc`(`t`)].incl( + getFieldIndex(`t`, `field`)) macro setDefaultValue*(t: typedesc, field: untyped, value: typed): typed = + ## Set the default value of an object field. Fields with default values may + ## be absent in YAML input when loading an instance of the object. If the + ## field is absent in the YAML input, the default value is assigned to the + ## field. + ## + ## Example usage: + ## + ## .. code-block:: + ## type MyObject = object + ## a, b: string + ## c: tuple[x, y: int] + ## setDefaultValue(MyObject, a, "foo") + ## setDefaultValue(MyObject, c, (1, 2)) let dSym = genSym(nskVar, ":default") nextBitvectorIndex = defaultVectors.len @@ -1424,6 +1464,16 @@ macro setDefaultValue*(t: typedesc, field: untyped, value: typed): typed = defaultVectors[`defaultBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`)) macro ignoreInputKey*(t: typedesc, name: string{lit}): typed = + ## Tell NimYAML that when loading an object of type ``t``, any mapping key + ## named ``name`` shall be ignored. This makes it possible to only load + ## relevant parts of a YAML input and ignoring other portions of the input. + ## + ## Example usage: + ## + ## .. code-block:: + ## type MyObject = object + ## a, b: string + ## ignoreInputKey(MyObject, "c") let nextIgnoredKeyList = ignoredKeyLists.len result = quote do: when not compiles(`ignoredKeyListProc`(`t`)):