From 12960b2e316f1fe593dc27b413c4a75de92dfacb Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Tue, 20 Sep 2016 21:53:38 +0200 Subject: [PATCH] Restructuring: no more includes --- private/events.nim | 73 --- private/internal.nim | 29 + private/stream.nim | 102 --- private/tagLibrary.nim | 83 --- test/{common.nim => commonTestUtils.nim} | 0 test/tdom.nim | 2 +- test/testEventParser.nim | 8 +- test/tjson.nim | 6 +- test/tserialization.nim | 92 +-- test/yamlTestSuite.nim | 4 +- yaml.nim | 752 +---------------------- yaml/common.nim | 35 ++ {private => yaml}/dom.nim | 28 +- {private => yaml}/hints.nim | 31 +- private/parse.nim => yaml/parser.nim | 77 ++- {private => yaml}/presenter.nim | 121 +++- {private => yaml}/serialization.nim | 115 +++- yaml/stream.nim | 281 +++++++++ yaml/taglib.nim | 290 +++++++++ private/json.nim => yaml/tojson.nim | 23 +- 20 files changed, 1049 insertions(+), 1103 deletions(-) delete mode 100644 private/events.nim create mode 100644 private/internal.nim delete mode 100644 private/stream.nim delete mode 100644 private/tagLibrary.nim rename test/{common.nim => commonTestUtils.nim} (100%) create mode 100644 yaml/common.nim rename {private => yaml}/dom.nim (90%) rename {private => yaml}/hints.nim (77%) rename private/parse.nim => yaml/parser.nim (89%) rename {private => yaml}/presenter.nim (82%) rename {private => yaml}/serialization.nim (89%) create mode 100644 yaml/stream.nim create mode 100644 yaml/taglib.nim rename private/json.nim => yaml/tojson.nim (83%) diff --git a/private/events.nim b/private/events.nim deleted file mode 100644 index 5be6147..0000000 --- a/private/events.nim +++ /dev/null @@ -1,73 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool = - if left.kind != right.kind: return false - case left.kind - of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlEndSeq: result = true - of yamlStartMap: - result = left.mapAnchor == right.mapAnchor and left.mapTag == right.mapTag - of yamlStartSeq: - result = left.seqAnchor == right.seqAnchor and left.seqTag == right.seqTag - of yamlScalar: - result = left.scalarAnchor == right.scalarAnchor and - left.scalarTag == right.scalarTag and - left.scalarContent == right.scalarContent - of yamlAlias: result = left.aliasTarget == right.aliasTarget - -proc `$`*(event: YamlStreamEvent): string = - result = $event.kind & '(' - case event.kind - of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard - of yamlStartMap: - result &= "tag=" & $event.mapTag - if event.mapAnchor != yAnchorNone: result &= ", anchor=" & $event.mapAnchor - of yamlStartSeq: - result &= "tag=" & $event.seqTag - if event.seqAnchor != yAnchorNone: result &= ", anchor=" & $event.seqAnchor - of yamlScalar: - result &= "tag=" & $event.scalarTag - if event.scalarAnchor != yAnchorNone: - result &= ", anchor=" & $event.scalarAnchor - result &= ", content=\"" & event.scalarContent & '\"' - of yamlAlias: - result &= "aliasTarget=" & $event.aliasTarget - result &= ")" - -proc tag*(event: YamlStreamEvent): TagId = - case event.kind - of yamlStartMap: result = event.mapTag - of yamlStartSeq: result = event.seqTag - of yamlScalar: result = event.scalarTag - else: raise newException(FieldError, "Event " & $event.kind & " has no tag") - -proc startDocEvent*(): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlStartDoc) - -proc endDocEvent*(): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlEndDoc) - -proc startMapEvent*(tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlStartMap, mapTag: tag, mapAnchor: anchor) - -proc endMapEvent*(): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlEndMap) - -proc startSeqEvent*(tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlStartSeq, seqTag: tag, seqAnchor: anchor) - -proc endSeqEvent*(): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlEndSeq) - -proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag, - scalarAnchor: anchor, scalarContent: content) - -proc aliasEvent*(anchor: AnchorId): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlAlias, aliasTarget: anchor) diff --git a/private/internal.nim b/private/internal.nim new file mode 100644 index 0000000..af74036 --- /dev/null +++ b/private/internal.nim @@ -0,0 +1,29 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +template internalError*(s: string) = + when not defined(release): + let ii = instantiationInfo() + echo "[NimYAML] Error in file ", ii.filename, " at line ", ii.line, ":" + echo s + when not defined(JS): + echo "[NimYAML] Stacktrace:" + try: writeStackTrace() + except: discard + echo "[NimYAML] Please report this bug." + quit 1 +template yAssert*(e: typed) = + when not defined(release): + if not e: + let ii = instantiationInfo() + echo "[NimYAML] Error in file ", ii.filename, " at line ", ii.line, ":" + echo "assertion failed!" + when not defined(JS): + echo "[NimYAML] Stacktrace:" + try: writeStackTrace() + except: discard + echo "[NimYAML] Please report this bug." + quit 1 \ No newline at end of file diff --git a/private/stream.nim b/private/stream.nim deleted file mode 100644 index d5b8553..0000000 --- a/private/stream.nim +++ /dev/null @@ -1,102 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -proc noLastContext(s: YamlStream, line, column: var int, - lineContent: var string): bool = - (line, column, lineContent) = (-1, -1, "") - result = false - -when not defined(JS): - type IteratorYamlStream = ref object of YamlStream - backend: iterator(): YamlStreamEvent - - proc initYamlStream*(backend: iterator(): YamlStreamEvent): YamlStream = - result = new(IteratorYamlStream) - result.peeked = false - result.isFinished = false - IteratorYamlStream(result).backend = backend - result.nextImpl = proc(s: YamlStream, e: var YamlStreamEvent): bool = - e = IteratorYamlStream(s).backend() - if finished(IteratorYamlStream(s).backend): - s.isFinished = true - result = false - else: result = true - result.lastTokenContextImpl = noLastContext - -type - BufferYamlStream = ref object of YamlStream - pos: int - buf: seq[YamlStreamEvent] not nil - -proc newBufferYamlStream(): BufferYamlStream not nil = - BufferYamlStream(peeked: false, isFinished: false, buf: @[], pos: 0, - lastTokenContextImpl: noLastContext, - nextImpl: proc(s: YamlStream, e: var YamlStreamEvent): bool = - let bys = BufferYamlStream(s) - if bys.pos == bys.buf.len: - result = false - s.isFinished = true - else: - e = bys.buf[bys.pos] - inc(bys.pos) - result = true - ) - -proc next*(s: YamlStream): YamlStreamEvent = - if s.peeked: - s.peeked = false - shallowCopy(result, s.cached) - return - else: - yAssert(not s.isFinished) - try: - while true: - if s.nextImpl(s, result): break - yAssert(not s.isFinished) - except YamlStreamError: - let cur = getCurrentException() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur.parent - raise e - except Exception: - let cur = getCurrentException() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur - raise e - -proc peek*(s: YamlStream): YamlStreamEvent = - if not s.peeked: - shallowCopy(s.cached, s.next()) - s.peeked = true - shallowCopy(result, s.cached) - -proc `peek=`*(s: YamlStream, value: YamlStreamEvent) = - s.cached = value - s.peeked = true - -proc finished*(s: YamlStream): bool = - if s.peeked: result = false - else: - try: - while true: - if s.isFinished: return true - if s.nextImpl(s, s.cached): - s.peeked = true - return false - except YamlStreamError: - let cur = getCurrentException() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur.parent - raise e - except Exception: - let cur = getCurrentException() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur - raise e - -proc getLastTokenContext*(s: YamlStream, line, column: var int, - lineContent: var string): bool = - result = s.lastTokenContextImpl(s, line, column, lineContent) diff --git a/private/tagLibrary.nim b/private/tagLibrary.nim deleted file mode 100644 index 874e9f6..0000000 --- a/private/tagLibrary.nim +++ /dev/null @@ -1,83 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -proc `$`*(id: TagId): string = - case id - of yTagQuestionMark: "?" - of yTagExclamationMark: "!" - of yTagString: "!!str" - of yTagSequence: "!!seq" - of yTagMapping: "!!map" - of yTagNull: "!!null" - of yTagBoolean: "!!bool" - of yTagInteger: "!!int" - of yTagFloat: "!!float" - of yTagOrderedMap: "!!omap" - of yTagPairs: "!!pairs" - of yTagSet: "!!set" - of yTagBinary: "!!binary" - of yTagMerge: "!!merge" - of yTagTimestamp: "!!timestamp" - of yTagValue: "!!value" - of yTagYaml: "!!yaml" - of yTagNimField: "!nim:field" - else: "<" & $cast[int](id) & ">" - -proc initTagLibrary*(): TagLibrary = - new(result) - result.tags = initTable[string, TagId]() - result.secondaryPrefix = yamlTagRepositoryPrefix - result.nextCustomTagId = yFirstCustomTagId - -proc registerUri*(tagLib: TagLibrary, uri: string): TagId = - tagLib.tags[uri] = tagLib.nextCustomTagId - result = tagLib.nextCustomTagId - tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1) - -proc uri*(tagLib: TagLibrary, id: TagId): string = - for iUri, iId in tagLib.tags.pairs: - if iId == id: return iUri - raise newException(KeyError, "Unknown tag id: " & $id) - -proc initFailsafeTagLibrary(): TagLibrary = - result = initTagLibrary() - result.tags["!"] = yTagExclamationMark - result.tags["?"] = yTagQuestionMark - result.tags["tag:yaml.org,2002:str"] = yTagString - result.tags["tag:yaml.org,2002:seq"] = yTagSequence - result.tags["tag:yaml.org,2002:map"] = yTagMapping - -proc initCoreTagLibrary(): TagLibrary = - result = initFailsafeTagLibrary() - result.tags["tag:yaml.org,2002:null"] = yTagNull - result.tags["tag:yaml.org,2002:bool"] = yTagBoolean - result.tags["tag:yaml.org,2002:int"] = yTagInteger - result.tags["tag:yaml.org,2002:float"] = yTagFloat - -proc initExtendedTagLibrary(): TagLibrary = - result = initCoreTagLibrary() - result.tags["tag:yaml.org,2002:omap"] = yTagOrderedMap - result.tags["tag:yaml.org,2002:pairs"] = yTagPairs - result.tags["tag:yaml.org,2002:binary"] = yTagBinary - result.tags["tag:yaml.org,2002:merge"] = yTagMerge - result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp - result.tags["tag:yaml.org,2002:value"] = yTagValue - result.tags["tag:yaml.org,2002:yaml"] = yTagYaml - -proc initSerializationTagLibrary(): TagLibrary = - result = initTagLibrary() - result.tags["!"] = yTagExclamationMark - result.tags["?"] = yTagQuestionMark - result.tags["tag:yaml.org,2002:str"] = yTagString - result.tags["tag:yaml.org,2002:null"] = yTagNull - result.tags["tag:yaml.org,2002:bool"] = yTagBoolean - result.tags["tag:yaml.org,2002:float"] = yTagFloat - result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp - result.tags["tag:yaml.org,2002:value"] = yTagValue - result.tags["tag:yaml.org,2002:binary"] = yTagBinary - result.tags["!nim:field"] = yTagNimField - result.tags["!nim:nil:string"] = yTagNimNilString - result.tags["!nim:nil:seq"] = yTagNimNilSeq diff --git a/test/common.nim b/test/commonTestUtils.nim similarity index 100% rename from test/common.nim rename to test/commonTestUtils.nim diff --git a/test/tdom.nim b/test/tdom.nim index ac5d4ac..2aa5c33 100644 --- a/test/tdom.nim +++ b/test/tdom.nim @@ -5,7 +5,7 @@ # distribution, for details about the copyright. import "../yaml" -import unittest, common +import unittest, commonTestUtils, streams suite "DOM": test "Composing simple Scalar": diff --git a/test/testEventParser.nim b/test/testEventParser.nim index 89796ce..f99e4a6 100644 --- a/test/testEventParser.nim +++ b/test/testEventParser.nim @@ -5,19 +5,19 @@ # distribution, for details about the copyright. import "../yaml" -import lexbase +import lexbase, streams, tables type LexerToken = enum plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, minusSeq, eqVal, eqAli, chevTag, andAnchor, quotContent, colonContent, noToken - + StreamPos = enum beforeStream, inStream, afterStream EventLexer = object of BaseLexer content: string - + EventStreamError = object of Exception proc nextToken(lex: var EventLexer): LexerToken = @@ -156,7 +156,7 @@ template setCurAnchor(val: AnchorId) = of yamlAlias: curEvent.aliasTarget = val else: raise newException(EventStreamError, $curEvent.kind & " may not have an anchor") - + template eventStart(k: YamlStreamEventKind) {.dirty.} = assertInStream() yieldEvent() diff --git a/test/tjson.nim b/test/tjson.nim index aeff7c4..4045766 100644 --- a/test/tjson.nim +++ b/test/tjson.nim @@ -6,7 +6,7 @@ import "../yaml" -import unittest, json +import unittest, json, streams proc wc(line, column: int, lineContent: string, message: string) = echo "Warning (", line, ",", column, "): ", message, "\n", lineContent @@ -31,10 +31,10 @@ proc ensureEqual(yamlIn, jsonIn: string) = suite "Constructing JSON": test "Simple Sequence": ensureEqual("- 1\n- 2\n- 3", "[1, 2, 3]") - + test "Simple Map": ensureEqual("a: b\nc: d", """{"a": "b", "c": "d"}""") - + test "Complex Structure": ensureEqual(""" %YAML 1.2 diff --git a/test/tserialization.nim b/test/tserialization.nim index 955e981..f54d228 100644 --- a/test/tserialization.nim +++ b/test/tserialization.nim @@ -5,31 +5,31 @@ # distribution, for details about the copyright. import "../yaml" -import unittest, strutils +import unittest, strutils, streams, tables type MyTuple = tuple str: string i: int32 b: bool - + TrafficLight = enum tlGreen, tlYellow, tlRed - + Person = object firstnamechar: char surname: string age: int32 - + Node = object value: string next: ref Node - + BetterInt = distinct int - + AnimalKind = enum akCat, akDog - + Animal = object name: string case kind: AnimalKind @@ -81,8 +81,8 @@ suite "Serialization": var input = newStringStream("-4247") var result: int load(input, result) - assert result == -4247, "result is " & $result - + assert result == -4247, "result is " & $result + input = newStringStream($(int64(int32.high) + 1'i64)) var gotException = false try: load(input, result) @@ -93,7 +93,7 @@ suite "Serialization": var input = -4247 var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual "%YAML 1.2\n--- \n\"-4247\"", output - + when sizeof(int) == sizeof(int64): input = int(int32.high) + 1 var gotException = false @@ -154,7 +154,7 @@ suite "Serialization": var result: string load(input, result) assert isNil(result) - + test "Dump nil string": let input: string = nil var output = dump(input, tsNone, asTidy, blockOnly) @@ -167,23 +167,23 @@ suite "Serialization": assert result.len == 2 assert result[0] == "a" assert result[1] == "b" - + test "Dump string sequence": var input = @["a", "b"] var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output - + test "Load nil seq": let input = newStringStream("!nim:nil:seq \"\"") var result: seq[int] load(input, result) assert isNil(result) - + test "Dump nil seq": let input: seq[int] = nil var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual "%YAML 1.2\n--- \n!nim:nil:seq \"\"", output - + test "Load char set": let input = newStringStream("- a\n- b") var result: set[char] @@ -191,12 +191,12 @@ suite "Serialization": assert result.card == 2 assert 'a' in result assert 'b' in result - + test "Dump char set": var input = {'a', 'b'} var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output - + test "Load array": let input = newStringStream("- 23\n- 42\n- 47") var result: array[0..2, int32] @@ -204,12 +204,12 @@ suite "Serialization": assert result[0] == 23 assert result[1] == 42 assert result[2] == 47 - + test "Dump array": let input = [23'i32, 42'i32, 47'i32] var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual "%YAML 1.2\n--- \n- 23\n- 42\n- 47", output - + test "Load Table[int, string]": let input = newStringStream("23: dreiundzwanzig\n42: zweiundvierzig") var result: Table[int32, string] @@ -217,7 +217,7 @@ suite "Serialization": assert result.len == 2 assert result[23] == "dreiundzwanzig" assert result[42] == "zweiundvierzig" - + test "Dump Table[int, string]": var input = initTable[int32, string]() input[23] = "dreiundzwanzig" @@ -225,7 +225,7 @@ suite "Serialization": var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual("%YAML 1.2\n--- \n23: dreiundzwanzig\n42: zweiundvierzig", output) - + test "Load OrderedTable[tuple[int32, int32], string]": let input = newStringStream("- {a: 23, b: 42}: drzw\n- {a: 13, b: 47}: drsi") var result: OrderedTable[tuple[a, b: int32], string] @@ -241,7 +241,7 @@ suite "Serialization": assert value == "drsi" else: assert false i.inc() - + test "Dump OrderedTable[tuple[int32, int32], string]": var input = initOrderedTable[tuple[a, b: int32], string]() input.add((a: 23'i32, b: 42'i32), "dreiundzwanzigzweiundvierzig") @@ -259,7 +259,7 @@ suite "Serialization": a: 13 b: 47 : dreizehnsiebenundvierzig""", output) - + test "Load Sequences in Sequence": let input = newStringStream(" - [1, 2, 3]\n - [4, 5]\n - [6]") var result: seq[seq[int32]] @@ -268,12 +268,12 @@ suite "Serialization": assert result[0] == @[1.int32, 2.int32, 3.int32] assert result[1] == @[4.int32, 5.int32] assert result[2] == @[6.int32] - + test "Dump Sequences in Sequence": let input = @[@[1.int32, 2.int32, 3.int32], @[4.int32, 5.int32], @[6.int32]] var output = dump(input, tsNone) assertStringEqual "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]", output - + test "Load Enum": let input = newStringStream("!nim:system:seq(tl)\n- !tl tlRed\n- tlGreen\n- tlYellow") @@ -283,12 +283,12 @@ suite "Serialization": assert result[0] == tlRed assert result[1] == tlGreen assert result[2] == tlYellow - + test "Dump Enum": let input = @[tlRed, tlGreen, tlYellow] var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual "%YAML 1.2\n--- \n- tlRed\n- tlGreen\n- tlYellow", output - + test "Load Tuple": let input = newStringStream("str: value\ni: 42\nb: true") var result: MyTuple @@ -316,7 +316,7 @@ suite "Serialization": loadMultiDoc(input, result) assert(result.len == 1) assert result[0] == 1 - + test "Load custom object": let input = newStringStream("firstnamechar: P\nsurname: Pan\nage: 12") var result: Person @@ -324,13 +324,13 @@ suite "Serialization": assert result.firstnamechar == 'P' assert result.surname == "Pan" assert result.age == 12 - + test "Dump custom object": let input = Person(firstnamechar: 'P', surname: "Pan", age: 12) var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual( "%YAML 1.2\n--- \nfirstnamechar: P\nsurname: Pan\nage: 12", output) - + test "Load sequence with explicit tags": let input = newStringStream("--- !nim:system:seq(" & "tag:yaml.org,2002:str)\n- !!str one\n- !!str two") @@ -338,13 +338,13 @@ suite "Serialization": load(input, result) assert result[0] == "one" assert result[1] == "two" - + test "Dump sequence with explicit tags": let input = @["one", "two"] var output = dump(input, tsAll, asTidy, blockOnly) assertStringEqual("%YAML 1.2\n--- !nim:system:seq(" & "tag:yaml.org,2002:str) \n- !!str one\n- !!str two", output) - + test "Load custom object with explicit root tag": let input = newStringStream( "--- !nim:custom:Person\nfirstnamechar: P\nsurname: Pan\nage: 12") @@ -353,14 +353,14 @@ suite "Serialization": assert result.firstnamechar == 'P' assert result.surname == "Pan" assert result.age == 12 - + test "Dump custom object with explicit root tag": let input = Person(firstnamechar: 'P', surname: "Pan", age: 12) var output = dump(input, tsRootOnly, asTidy, blockOnly) assertStringEqual("%YAML 1.2\n" & "--- !nim:custom:Person \nfirstnamechar: P\nsurname: Pan\nage: 12", output) - + test "Load custom variant object": let input = newStringStream( "---\n- - name: Bastet\n - kind: akCat\n - purringIntensity: 7\n" & @@ -374,7 +374,7 @@ suite "Serialization": assert result[1].name == "Anubis" assert result[1].kind == akDog assert result[1].barkometer == 13 - + test "Dump custom variant object": let input = @[Animal(name: "Bastet", kind: akCat, purringIntensity: 7), Animal(name: "Anubis", kind: akDog, barkometer: 13)] @@ -395,7 +395,7 @@ suite "Serialization": kind: akDog - barkometer: 13""", output - + test "Dump cyclic data structure": var a = newNode("a") @@ -413,15 +413,15 @@ next: next: value: c next: *a""", output - + test "Load cyclic data structure": let input = newStringStream("""%YAML 1.2 ---- !nim:system:seq(example.net:Node) -- &a +--- !nim:system:seq(example.net:Node) +- &a value: a - next: &b + next: &b value: b - next: &c + next: &c value: c next: *a - *b @@ -442,7 +442,7 @@ next: assert(result[0].next == result[1]) assert(result[1].next == result[2]) assert(result[2].next == result[0]) - + test "Load nil values": let input = newStringStream("- ~\n- !!str ~") var result: seq[ref string] @@ -452,11 +452,11 @@ next: echo "line ", ex.line, ", column ", ex.column, ": ", ex.msg echo ex.lineContent raise ex - + assert(result.len == 2) assert(result[0] == nil) assert(result[1][] == "~") - + test "Dump nil values": var input = newSeq[ref string]() input.add(nil) @@ -466,7 +466,7 @@ next: assertStringEqual( "%YAML 1.2\n--- !nim:system:seq(tag:yaml.org,2002:str) \n- !!null ~\n- !!str ~", output) - + test "Custom constructObject": let input = newStringStream("- 1\n- !test:BetterInt 2") var result: seq[BetterInt] @@ -474,7 +474,7 @@ next: assert(result.len == 2) assert(result[0] == 2.BetterInt) assert(result[1] == 3.BetterInt) - + test "Custom representObject": let input = @[1.BetterInt, 9998887.BetterInt, 98312.BetterInt] var output = dump(input, tsAll, asTidy, blockOnly) diff --git a/test/yamlTestSuite.nim b/test/yamlTestSuite.nim index 4824618..2423af4 100644 --- a/test/yamlTestSuite.nim +++ b/test/yamlTestSuite.nim @@ -4,8 +4,8 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. -import os, terminal, strutils -import testEventParser, common +import os, terminal, strutils, streams +import testEventParser, commonTestUtils import "../yaml" const gitCmd = diff --git a/yaml.nim b/yaml.nim index 3bf4a87..ffa5f0c 100644 --- a/yaml.nim +++ b/yaml.nim @@ -16,751 +16,7 @@ ## automatically supported. While JSON is less readable than YAML, ## this enhances interoperability with other languages. -import streams, unicode, lexbase, tables, strutils, json, hashes, queues, - macros, typetraits, parseutils, private/lex -export streams, tables, json - -when defined(yamlDebug): import terminal - -type - TypeHint* = enum - ## A type hint can be computed from scalar content and tells you what - ## NimYAML thinks the scalar's type is. It is generated by - ## `guessType <#guessType,string>`_ The first matching RegEx - ## in the following table will be the type hint of a scalar string. - ## - ## You can use it to determine the type of YAML scalars that have a '?' - ## non-specific tag, but using this feature is completely optional. - ## - ## ================== ========================= - ## Name RegEx - ## ================== ========================= - ## ``yTypeInteger`` ``0 | -? [1-9] [0-9]*`` - ## ``yTypeFloat`` ``-? [1-9] ( \. [0-9]* [1-9] )? ( e [-+] [1-9] [0-9]* )?`` - ## ``yTypeFloatInf`` ``-? \. (inf | Inf | INF)`` - ## ``yTypeFloatNaN`` ``-? \. (nan | NaN | NAN)`` - ## ``yTypeBoolTrue`` ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON`` - ## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF`` - ## ``yTypeNull`` ``~ | null | Null | NULL`` - ## ``yTypeUnknown`` ``*`` - ## ================== ========================= - yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue, - yTypeBoolFalse, yTypeNull, yTypeUnknown - - YamlStreamEventKind* = enum - ## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds - ## are discussed in `YamlStreamEvent <#YamlStreamEvent>`_. - yamlStartDoc, yamlEndDoc, yamlStartMap, yamlEndMap, - yamlStartSeq, yamlEndSeq, yamlScalar, yamlAlias - - TagId* = distinct int ## \ - ## A ``TagId`` identifies a tag URI, like for example - ## ``"tag:yaml.org,2002:str"``. The URI corresponding to a ``TagId`` can - ## be queried from the `TagLibrary <#TagLibrary>`_ which was - ## used to create this ``TagId``; e.g. when you parse a YAML character - ## stream, the ``TagLibrary`` of the parser is the one which generates - ## the resulting ``TagId`` s. - ## - ## URI strings are mapped to ``TagId`` s for efficiency reasons (you - ## do not need to compare strings every time) and to be able to - ## discover unknown tag URIs early in the parsing process. - - AnchorId* = distinct int ## \ - ## An ``AnchorId`` identifies an anchor in the current document. It - ## becomes invalid as soon as the current document scope is invalidated - ## (for example, because the parser yielded a ``yamlEndDocument`` - ## event). ``AnchorId`` s exists because of efficiency, much like - ## ``TagId`` s. The actual anchor name is a presentation detail and - ## cannot be queried by the user. - - YamlStreamEvent* = object - ## An element from a `YamlStream <#YamlStream>`_. Events that start an - ## object (``yamlStartMap``, ``yamlStartSeq``, ``yamlScalar``) have - ## an optional anchor and a tag associated with them. The anchor will be - ## set to ``yAnchorNone`` if it doesn't exist. - ## - ## A non-existing tag in the YAML character stream will be resolved to - ## the non-specific tags ``?`` or ``!`` according to the YAML - ## specification. These are by convention mapped to the ``TagId`` s - ## ``yTagQuestionMark`` and ``yTagExclamationMark`` respectively. - ## Mapping is done by a `TagLibrary <#TagLibrary>`_. - case kind*: YamlStreamEventKind - of yamlStartMap: - mapAnchor* : AnchorId - mapTag* : TagId - of yamlStartSeq: - seqAnchor* : AnchorId - seqTag* : TagId - of yamlScalar: - scalarAnchor* : AnchorId - scalarTag* : TagId - scalarContent*: string # may not be nil (but empty) - of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard - of yamlAlias: - aliasTarget* : AnchorId - - YamlStream* = ref object of RootObj ## \ - ## A ``YamlStream`` is an iterator-like object that yields a - ## well-formed stream of ``YamlStreamEvents``. Well-formed means that - ## every ``yamlStartMap`` is terminated by a ``yamlEndMap``, every - ## ``yamlStartSeq`` is terminated by a ``yamlEndSeq`` and every - ## ``yamlStartDoc`` is terminated by a ``yamlEndDoc``. Moreover, every - ## emitted mapping has an even number of children. - ## - ## The creator of a ``YamlStream`` is responsible for it being - ## well-formed. A user of the stream may assume that it is well-formed - ## and is not required to check for it. The procs in this module will - ## always yield a well-formed ``YamlStream`` and expect it to be - ## well-formed if they take it as input parameter. - nextImpl: proc(s: YamlStream, e: var YamlStreamEvent): bool - lastTokenContextImpl: - proc(s: YamlStream, line, column: var int, - lineContent: var string): bool {.raises: [].} - isFinished: bool - peeked: bool - cached: YamlStreamEvent - - TagLibrary* = ref object - ## A ``TagLibrary`` maps tag URIs to ``TagId`` s. - ## - ## When `YamlParser <#YamlParser>`_ encounters tags not existing in the - ## tag library, it will use - ## `registerUri <#registerUri,TagLibrary,string>`_ to add - ## the tag to the library. - ## - ## You can base your tag library on common tag libraries by initializing - ## them with `initFailsafeTagLibrary <#initFailsafeTagLibrary>`_, - ## `initCoreTagLibrary <#initCoreTagLibrary>`_ or - ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_. - tags*: Table[string, TagId] - nextCustomTagId*: TagId - secondaryPrefix*: string - - - WarningCallback* = proc(line, column: int, lineContent: string, - message: string) - ## Callback for parser warnings. Currently, this callback may be called - ## on two occasions while parsing a YAML document stream: - ## - ## - If the version number in the ``%YAML`` directive does not match - ## ``1.2``. - ## - If there is an unknown directive encountered. - - YamlParser* = ref object - ## A parser object. Retains its ``TagLibrary`` across calls to - ## `parse <#parse,YamlParser,Stream>`_. Can be used - ## to access anchor names while parsing a YAML character stream, but - ## only until the document goes out of scope (i.e. until - ## ``yamlEndDocument`` is yielded). - tagLib: TagLibrary - callback: WarningCallback - anchors: Table[string, AnchorId] - - PresentationStyle* = enum - ## Different styles for YAML character stream output. - ## - ## - ``ypsMinimal``: Single-line flow-only output which tries to - ## use as few characters as possible. - ## - ``ypsCanonical``: Canonical YAML output. Writes all tags except - ## for the non-specific tags ``?`` and ``!``, uses flow style, quotes - ## all string scalars. - ## - ``ypsDefault``: Tries to be as human-readable as possible. Uses - ## block style by default, but tries to condense mappings and - ## sequences which only contain scalar nodes into a single line using - ## flow style. - ## - ``ypsJson``: Omits the ``%YAML`` directive and the ``---`` - ## marker. Uses flow style. Flattens anchors and aliases, omits tags. - ## Output will be parseable as JSON. ``YamlStream`` to dump may only - ## contain one document. - ## - ``ypsBlockOnly``: Formats all output in block style, does not use - ## flow style at all. - psMinimal, psCanonical, psDefault, psJson, psBlockOnly - - TagStyle* = enum - ## Whether object should be serialized with explicit tags. - ## - ## - ``tsNone``: No tags will be outputted unless necessary. - ## - ``tsRootOnly``: A tag will only be outputted for the root tag and - ## where necessary. - ## - ``tsAll``: Tags will be outputted for every object. - tsNone, tsRootOnly, tsAll - - AnchorStyle* = enum - ## How ref object should be serialized. - ## - ## - ``asNone``: No anchors will be outputted. Values present at - ## multiple places in the content that should be serialized will be - ## fully serialized at every occurence. If the content is cyclic, this - ## will lead to an endless loop! - ## - ``asTidy``: Anchors will only be generated for objects that - ## actually occur more than once in the content to be serialized. - ## This is a bit slower and needs more memory than ``asAlways``. - ## - ``asAlways``: Achors will be generated for every ref object in the - ## content to be serialized, regardless of whether the object is - ## referenced again afterwards - asNone, asTidy, asAlways - - NewLineStyle* = enum - ## What kind of newline sequence is used when presenting. - ## - ## - ``nlLF``: Use a single linefeed char as newline. - ## - ``nlCRLF``: Use a sequence of carriage return and linefeed as - ## newline. - ## - ``nlOSDefault``: Use the target operation system's default newline - ## sequence (CRLF on Windows, LF everywhere else). - nlLF, nlCRLF, nlOSDefault - - OutputYamlVersion* = enum - ## Specify which YAML version number the presenter shall emit. The - ## presenter will always emit content that is valid YAML 1.1, but by - ## default will write a directive ``%YAML 1.2``. For compatibility with - ## other YAML implementations, it is possible to change this here. - ## - ## It is also possible to specify that the presenter shall not emit any - ## YAML version. The generated content is then guaranteed to be valid - ## YAML 1.1 and 1.2 (but not 1.0 or any newer YAML version). - ov1_2, ov1_1, ovNone - - PresentationOptions* = object - ## Options for generating a YAML character stream - style*: PresentationStyle - indentationStep*: int - newlines*: NewLineStyle - outputVersion*: OutputYamlVersion - - ConstructionContext* = ref object - ## Context information for the process of constructing Nim values from YAML. - refs: Table[AnchorId, pointer] - - SerializationContext* = ref object - ## Context information for the process of serializing YAML from Nim values. - refs: Table[pointer, AnchorId] - style: AnchorStyle - nextAnchorId: AnchorId - put*: proc(e: YamlStreamEvent) {.raises: [], closure.} - - YamlNodeKind* = enum - yScalar, yMapping, ySequence - - YamlNode* = ref YamlNodeObj not nil - ## Represents a node in a ``YamlDocument``. - - YamlNodeObj* = object - tag*: string - case kind*: YamlNodeKind - of yScalar: content*: string - of ySequence: children*: seq[YamlNode] - of yMapping: pairs*: seq[tuple[key, value: YamlNode]] - - YamlDocument* = object - ## Represents a YAML document. - root*: YamlNode - - YamlLoadingError* = object of Exception - ## Base class for all exceptions that may be raised during the process - ## of loading a YAML character stream. - line*: int ## line number (1-based) where the error was encountered - column*: int ## column number (1-based) where the error was encountered - lineContent*: string ## \ - ## content of the line where the error was encountered. Includes a - ## second line with a marker ``^`` at the position where the error - ## was encountered. - - YamlParserError* = object of YamlLoadingError - ## A parser error is raised if the character stream that is parsed is - ## not a valid YAML character stream. This stream cannot and will not be - ## parsed wholly nor partially and all events that have been emitted by - ## the YamlStream the parser provides should be discarded. - ## - ## A character stream is invalid YAML if and only if at least one of the - ## following conditions apply: - ## - ## - There are invalid characters in an element whose contents is - ## restricted to a limited set of characters. For example, there are - ## characters in a tag URI which are not valid URI characters. - ## - An element has invalid indentation. This can happen for example if - ## a block list element indicated by ``"- "`` is less indented than - ## the element in the previous line, but there is no block sequence - ## list open at the same indentation level. - ## - The YAML structure is invalid. For example, an explicit block map - ## indicated by ``"? "`` and ``": "`` may not suddenly have a block - ## sequence item (``"- "``) at the same indentation level. Another - ## possible violation is closing a flow style object with the wrong - ## closing character (``}``, ``]``) or not closing it at all. - ## - A custom tag shorthand is used that has not previously been - ## declared with a ``%TAG`` directive. - ## - Multiple tags or anchors are defined for the same node. - ## - An alias is used which does not map to any anchor that has - ## previously been declared in the same document. - ## - An alias has a tag or anchor associated with it. - ## - ## Some elements in this list are vague. For a detailed description of a - ## valid YAML character stream, see the YAML specification. - - YamlPresenterJsonError* = object of Exception - ## Exception that may be raised by the YAML presenter when it is - ## instructed to output JSON, but is unable to do so. This may occur if: - ## - ## - The given `YamlStream <#YamlStream>`_ contains a map which has any - ## non-scalar type as key. - ## - Any float scalar bears a ``NaN`` or positive/negative infinity value - - YamlPresenterOutputError* = object of Exception - ## Exception that may be raised by the YAML presenter. This occurs if - ## writing character data to the output stream raises any exception. - ## The error that has occurred is available from ``parent``. - - YamlStreamError* = object of Exception - ## Exception that may be raised by a ``YamlStream`` when the underlying - ## backend raises an exception. The error that has occurred is - ## available from ``parent``. - - YamlConstructionError* = object of YamlLoadingError - ## Exception that may be raised when constructing data objects from a - ## `YamlStream <#YamlStream>`_. The fields ``line``, ``column`` and - ## ``lineContent`` are only available if the costructing proc also does - ## parsing, because otherwise this information is not available to the - ## costruction proc. - -const - # failsafe schema - - yTagExclamationMark*: TagId = 0.TagId ## ``!`` non-specific tag - yTagQuestionMark* : TagId = 1.TagId ## ``?`` non-specific tag - yTagString* : TagId = 2.TagId ## \ - ## `!!str `_ tag - yTagSequence* : TagId = 3.TagId ## \ - ## `!!seq `_ tag - yTagMapping* : TagId = 4.TagId ## \ - ## `!!map `_ tag - - # json & core schema - - yTagNull* : TagId = 5.TagId ## \ - ## `!!null `_ tag - yTagBoolean* : TagId = 6.TagId ## \ - ## `!!bool `_ tag - yTagInteger* : TagId = 7.TagId ## \ - ## `!!int `_ tag - yTagFloat* : TagId = 8.TagId ## \ - ## `!!float `_ tag - - # other language-independent YAML types (from http://yaml.org/type/ ) - - yTagOrderedMap* : TagId = 9.TagId ## \ - ## `!!omap `_ tag - yTagPairs* : TagId = 10.TagId ## \ - ## `!!pairs `_ tag - yTagSet* : TagId = 11.TagId ## \ - ## `!!set `_ tag - yTagBinary* : TagId = 12.TagId ## \ - ## `!!binary `_ tag - yTagMerge* : TagId = 13.TagId ## \ - ## `!!merge `_ tag - yTagTimestamp* : TagId = 14.TagId ## \ - ## `!!timestamp `_ tag - yTagValue* : TagId = 15.TagId ## \ - ## `!!value `_ tag - yTagYaml* : TagId = 16.TagId ## \ - ## `!!yaml `_ tag - - yTagNimField* : TagId = 100.TagId ## \ - ## This tag is used in serialization for the name of a field of an - ## object. It may contain any string scalar that is a valid Nim symbol. - - yTagNimNilString* : TagId = 101.TagId ## for strings that are nil - yTagNimNilSeq* : TagId = 102.TagId ## \ - ## for seqs that are nil. This tag is used regardless of the seq's generic - ## type parameter. - - yFirstCustomTagId* : TagId = 1000.TagId ## \ - ## The first ``TagId`` which should be assigned to an URI that does not - ## exist in the ``YamlTagLibrary`` which is used for parsing. - - yAnchorNone*: AnchorId = (-1).AnchorId ## \ - ## yielded when no anchor was defined for a YAML node - - yamlTagRepositoryPrefix* = "tag:yaml.org,2002:" - - defaultPresentationOptions* = - PresentationOptions(style: psDefault, indentationStep: 2, - newlines: nlOSDefault) - -# used throughout implementation code, therefore defined here -template internalError(s: string) = - when not defined(release): - let ii = instantiationInfo() - echo "[NimYAML] Error in file ", ii.filename, " at line ", ii.line, ":" - echo s - when not defined(JS): - echo "[NimYAML] Stacktrace:" - try: writeStackTrace() - except: discard - echo "[NimYAML] Please report this bug." - quit 1 -template yAssert(e: typed) = - when not defined(release): - if not e: - let ii = instantiationInfo() - echo "[NimYAML] Error in file ", ii.filename, " at line ", ii.line, ":" - echo "assertion failed!" - when not defined(JS): - echo "[NimYAML] Stacktrace:" - try: writeStackTrace() - except: discard - echo "[NimYAML] Please report this bug." - quit 1 - -# interface - -proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool {.raises: [].} - ## compares all existing fields of the given items - -proc `$`*(event: YamlStreamEvent): string {.raises: [].} - ## outputs a human-readable string describing the given event - -proc tag*(event: YamlStreamEvent): TagId {.raises: [FieldError].} - -proc startDocEvent*(): YamlStreamEvent {.inline, raises: [].} -proc endDocEvent*(): YamlStreamEvent {.inline, raises: [].} -proc startMapEvent*(tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): - YamlStreamEvent {.inline, raises: [].} -proc endMapEvent*(): YamlStreamEvent {.inline, raises: [].} -proc startSeqEvent*(tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): - YamlStreamEvent {.inline, raises: [].} -proc endSeqEvent*(): YamlStreamEvent {.inline, raises: [].} -proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): - YamlStreamEvent {.inline, raises: [].} -proc aliasEvent*(anchor: AnchorId): YamlStreamEvent {.inline, raises: [].} - -proc `==`*(left, right: TagId): bool {.borrow.} -proc `$`*(id: TagId): string -proc hash*(id: TagId): Hash {.borrow.} - -proc `==`*(left, right: AnchorId): bool {.borrow.} -proc `$`*(id: AnchorId): string {.borrow.} -proc hash*(id: AnchorId): Hash {.borrow.} - -when not defined(JS): - proc initYamlStream*(backend: iterator(): YamlStreamEvent): - YamlStream {.raises: [].} - ## Creates a new ``YamlStream`` that uses the given iterator as backend. -proc next*(s: YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} - ## Get the next item of the stream. Requires ``finished(s) == true``. - ## If the backend yields an exception, that exception will be encapsulated - ## into a ``YamlStreamError``, which will be raised. -proc peek*(s: YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} - ## Get the next item of the stream without advancing the stream. - ## Requires ``finished(s) == true``. Handles exceptions of the backend like - ## ``next()``. -proc `peek=`*(s: YamlStream, value: YamlStreamEvent) {.raises: [].} - ## Set the next item of the stream. Will replace a previously peeked item, - ## if one exists. -proc finished*(s: YamlStream): bool {.raises: [YamlStreamError].} - ## ``true`` if no more items are available in the stream. Handles exceptions - ## of the backend like ``next()``. -proc getLastTokenContext*(s: YamlStream, line, column: var int, - lineContent: var string): bool - ## ``true`` if source context information is available about the last returned - ## token. If ``true``, line, column and lineContent are set to position and - ## line content where the last token has been read from. -iterator items*(s: YamlStream): YamlStreamEvent - {.raises: [YamlStreamError].} = - ## Iterate over all items of the stream. You may not use ``peek()`` on the - ## stream while iterating. - while not s.finished(): yield s.next() - -proc initTagLibrary*(): TagLibrary {.raises: [].} - ## initializes the ``tags`` table and sets ``nextCustomTagId`` to - ## ``yFirstCustomTagId``. - -proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} - ## registers a custom tag URI with a ``TagLibrary``. The URI will get - ## the ``TagId`` ``nextCustomTagId``, which will be incremented. - -proc uri*(tagLib: TagLibrary, id: TagId): string {.raises: [KeyError].} - ## retrieve the URI a ``TagId`` maps to. - -proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} - ## Contains only: - ## - ``!`` - ## - ``?`` - ## - ``!!str`` - ## - ``!!map`` - ## - ``!!seq`` -proc initCoreTagLibrary*(): TagLibrary {.raises: [].} - ## Contains everything in ``initFailsafeTagLibrary`` plus: - ## - ``!!null`` - ## - ``!!bool`` - ## - ``!!int`` - ## - ``!!float`` -proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} - ## Contains everything from ``initCoreTagLibrary`` plus: - ## - ``!!omap`` - ## - ``!!pairs`` - ## - ``!!set`` - ## - ``!!binary`` - ## - ``!!merge`` - ## - ``!!timestamp`` - ## - ``!!value`` - ## - ``!!yaml`` - -proc initSerializationTagLibrary(): TagLibrary {.raises: [].} - -proc guessType*(scalar: string): TypeHint {.raises: [].} - ## Parse scalar string according to the RegEx table documented at - ## `TypeHint <#TypeHind>`_. - -proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(), - callback: WarningCallback = nil): YamlParser {.raises: [].} - ## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called - ## whenever the parser yields a warning. - -proc parse*(p: YamlParser, s: Stream): YamlStream {.raises: [YamlParserError].} - ## Parse the given stream as YAML character stream. - ## The only Exception that can be raised comes from opening the Stream. - -proc parse*(p: YamlParser, str: string): YamlStream - {.raises: [YamlParserError].} - -proc defineOptions*(style: PresentationStyle = psDefault, - indentationStep: int = 2, - newlines: NewLineStyle = nlOSDefault, - outputVersion: OutputYamlVersion = ov1_2): - PresentationOptions {.raises: [].} - ## Define a set of options for presentation. Convenience proc that requires - ## you to only set those values that should not equal the default. - -proc constructJson*(s: var YamlStream): seq[JsonNode] - {.raises: [YamlConstructionError, YamlStreamError].} - ## Construct an in-memory JSON tree from a YAML event stream. The stream may - ## not contain any tags apart from those in ``coreTagLibrary``. Anchors and - ## aliases will be resolved. Maps in the input must not contain - ## non-scalars as keys. Each element of the result represents one document - ## in the YAML stream. - ## - ## **Warning:** The special float values ``[+-]Inf`` and ``NaN`` will be - ## parsed into Nim's JSON structure without error. However, they cannot be - ## rendered to a JSON character stream, because these values are not part - ## of the JSON specification. Nim's JSON implementation currently does not - ## check for these values and will output invalid JSON when rendering one - ## of these values into a JSON character stream. - -proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [YamlParserError].} - ## Uses `YamlParser <#YamlParser>`_ and - ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree - ## from a YAML character stream. - -proc present*(s: var YamlStream, target: Stream, - tagLib: TagLibrary, - options: PresentationOptions = defaultPresentationOptions) - {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, - YamlStreamError].} - ## Convert ``s`` to a YAML character stream and write it to ``target``. - -proc present*(s: var YamlStream, tagLib: TagLibrary, - options: PresentationOptions = defaultPresentationOptions): - string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, - YamlStreamError].} - ## Convert ``s`` to a YAML character stream and return it as string. - -proc transform*(input: Stream | string, output: Stream, - options: PresentationOptions = defaultPresentationOptions) - {.raises: [IOError, YamlParserError, YamlPresenterJsonError, - YamlPresenterOutputError].} - ## Parser ``input`` as YAML character stream and then dump it to ``output`` - ## while resolving non-specific tags to the ones in the YAML core tag - ## library. - -proc transform*(input: Stream | string, - options: PresentationOptions = defaultPresentationOptions): - string {.raises: [IOError, YamlParserError, YamlPresenterJsonError, - YamlPresenterOutputError].} - ## Parser ``input`` as YAML character stream, resolves non-specific tags to - ## the ones in the YAML core tag library, and then returns a serialized - ## YAML string that represents the stream. - -proc constructChild*[T](s: var YamlStream, c: ConstructionContext, - result: var T) - {.raises: [YamlConstructionError, YamlStreamError].} - ## Constructs an arbitrary Nim value from a part of a YAML stream. - ## The stream will advance until after the finishing token that was used - ## for constructing the value. The ``ConstructionContext`` is needed for - ## potential child objects which may be refs. - -proc constructChild*(s: var YamlStream, c: ConstructionContext, - result: var string) - {.raises: [YamlConstructionError, YamlStreamError].} - ## Constructs a Nim value that is a string from a part of a YAML stream. - ## This specialization takes care of possible nil strings. - -proc constructChild*[T](s: var YamlStream, c: ConstructionContext, - result: var seq[T]) - {.raises: [YamlConstructionError, YamlStreamError].} - ## Constructs a Nim value that is a string from a part of a YAML stream. - ## This specialization takes care of possible nil seqs. - -proc constructChild*[O](s: var YamlStream, c: ConstructionContext, - result: var ref O) - {.raises: [YamlStreamError].} - ## Constructs an arbitrary Nim value from a part of a YAML stream. - ## The stream will advance until after the finishing token that was used - ## for constructing the value. The object may be constructed from an alias - ## node which will be resolved using the ``ConstructionContext``. - -proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext) - {.raises: [YamlStreamError].} - ## Represents an arbitrary Nim reference value as YAML object. The object - ## may be represented as alias node if it is already present in the - ## ``SerializationContext``. - -proc representChild*(value: string, ts: TagStyle, c: SerializationContext) - {.inline, raises: [].} - ## Represents a Nim string. Supports nil strings. - -proc representChild*[O](value: O, ts: TagStyle, - c: SerializationContext) {.raises: [YamlStreamError].} - ## Represents an arbitrary Nim object as YAML object. - -proc construct*[T](s: var YamlStream, target: var T) - {.raises: [YamlStreamError].} - ## Constructs a Nim value from a YAML stream. - -proc load*[K](input: Stream | string, target: var K) - {.raises: [YamlConstructionError, IOError, YamlParserError].} - ## Loads a Nim value from a YAML character stream. - -proc represent*[T](value: T, ts: TagStyle = tsRootOnly, - a: AnchorStyle = asTidy): YamlStream - {.raises: [YamlStreamError].} - ## Represents a Nim value as ``YamlStream`` - -proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly, - anchorStyle: AnchorStyle = asTidy, - options: PresentationOptions = defaultPresentationOptions) - {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, - YamlStreamError].} - ## Dump a Nim value as YAML character stream. - -proc dump*[K](value: K, tagStyle: TagStyle = tsRootOnly, - anchorStyle: AnchorStyle = asTidy, - options: PresentationOptions = defaultPresentationOptions): - string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, - YamlStreamError].} - ## Dump a Nim value as YAML into a string - -var - serializationTagLibrary* = initSerializationTagLibrary() ## \ - ## contains all local tags that are used for type serialization. Does - ## not contain any of the specific default tags for sequences or maps, - ## as those are not suited for Nim's static type system. - ## - ## Should not be modified manually. Will be extended by - ## `serializable <#serializable,stmt,stmt>`_. - -var - nextStaticTagId {.compileTime.} = 100.TagId ## \ - ## used for generating unique TagIds with ``setTagUri``. - registeredUris {.compileTime.} = newSeq[string]() ## \ - ## Since Table doesn't really work at compile time, we also store - ## registered URIs here to be able to generate a static compiler error - ## when the user tries to register an URI more than once. - -template setTagUri*(t: typedesc, uri: string): typed = - ## Associate the given uri with a certain type. This uri is used as YAML tag - ## when loading and dumping values of this type. - when uri in registeredUris: - {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .} - const id {.genSym.} = nextStaticTagId - static: - registeredUris.add(uri) - nextStaticTagId = TagId(int(nextStaticTagId) + 1) - when nextStaticTagId == yFirstCustomTagId: - {.fatal: "Too many tags!".} - serializationTagLibrary.tags[uri] = id - proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = id - ## autogenerated - -template setTagUri*(t: typedesc, uri: string, idName: untyped): typed = - ## Like `setTagUri <#setTagUri.t,typedesc,string>`_, but lets - ## you choose a symbol for the `TagId <#TagId>`_ of the uri. This is only - ## necessary if you want to implement serialization / construction yourself. - when uri in registeredUris: - {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .} - const idName* = nextStaticTagId - static: - registeredUris.add(uri) - nextStaticTagId = TagId(int(nextStaticTagId) + 1) - serializationTagLibrary.tags[uri] = idName - proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName - ## autogenerated - -proc canBeImplicit(t: typedesc): bool {.compileTime.} = - let tDesc = getType(t) - if tDesc.kind != nnkObjectTy: return false - if tDesc[2].len != 1: return false - if tDesc[2][0].kind != nnkRecCase: return false - var foundEmptyBranch = false - for i in 1.. tDesc[2][0].len - 1: - case tDesc[2][0][i][1].len # branch contents - of 0: - if foundEmptyBranch: return false - else: foundEmptyBranch = true - of 1: discard - else: return false - return true - -template markAsImplicit*(t: typedesc): typed = - ## Mark a variant object type as implicit. This requires the type to consist - ## of nothing but a case expression and each branch of the case expression - ## containing exactly one field - with the exception that one branch may - ## contain zero fields. - when canBeImplicit(t): - # this will be checked by means of compiles(implicitVariantObject(...)) - proc implicitVariantObject*(unused: t) = discard - else: - {. fatal: "This type cannot be marked as implicit" .} - -static: - # standard YAML tags used by serialization - registeredUris.add("!") - registeredUris.add("?") - registeredUris.add("tag:yaml.org,2002:str") - registeredUris.add("tag:yaml.org,2002:null") - registeredUris.add("tag:yaml.org,2002:bool") - registeredUris.add("tag:yaml.org,2002:float") - registeredUris.add("tag:yaml.org,2002:timestamp") - registeredUris.add("tag:yaml.org,2002:value") - registeredUris.add("tag:yaml.org,2002:binary") - # special tags used by serialization - registeredUris.add("!nim:field") - registeredUris.add("!nim:nil:string") - registeredUris.add("!nim:nil:seq") - -# tags for Nim's standard types -setTagUri(char, "!nim:system:char", yTagNimChar) -setTagUri(int8, "!nim:system:int8", yTagNimInt8) -setTagUri(int16, "!nim:system:int16", yTagNimInt16) -setTagUri(int32, "!nim:system:int32", yTagNimInt32) -setTagUri(int64, "!nim:system:int64", yTagNimInt64) -setTagUri(uint8, "!nim:system:uint8", yTagNimUInt8) -setTagUri(uint16, "!nim:system:uint16", yTagNimUInt16) -setTagUri(uint32, "!nim:system:uint32", yTagNimUInt32) -setTagUri(uint64, "!nim:system:uint64", yTagNimUInt64) -setTagUri(float32, "!nim:system:float32", yTagNimFloat32) -setTagUri(float64, "!nim:system:float64", yTagNimFloat64) - -# implementation - -include private.tagLibrary -include private.events -include private.json -include private.stream -include private.presenter -include private.parse -include private.hints -include private.serialization -include private.dom +import yaml.common, yaml.dom, yaml.hints, yaml.parser, yaml.presenter, + yaml.serialization, yaml.stream, yaml.taglib, yaml.tojson +export yaml.common, yaml.dom, yaml.hints, yaml.parser, yaml.presenter, + yaml.serialization, yaml.stream, yaml.taglib, yaml.tojson \ No newline at end of file diff --git a/yaml/common.nim b/yaml/common.nim new file mode 100644 index 0000000..f0bbeee --- /dev/null +++ b/yaml/common.nim @@ -0,0 +1,35 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +import hashes + +type + TagId* = distinct int ## \ + ## A ``TagId`` identifies a tag URI, like for example + ## ``"tag:yaml.org,2002:str"``. The URI corresponding to a ``TagId`` can + ## be queried from the `TagLibrary <#TagLibrary>`_ which was + ## used to create this ``TagId``; e.g. when you parse a YAML character + ## stream, the ``TagLibrary`` of the parser is the one which generates + ## the resulting ``TagId`` s. + ## + ## URI strings are mapped to ``TagId`` s for efficiency reasons (you + ## do not need to compare strings every time) and to be able to + ## discover unknown tag URIs early in the parsing process. + + AnchorId* = distinct int ## \ + ## An ``AnchorId`` identifies an anchor in the current document. It + ## becomes invalid as soon as the current document scope is invalidated + ## (for example, because the parser yielded a ``yamlEndDocument`` + ## event). ``AnchorId`` s exists because of efficiency, much like + ## ``TagId`` s. The actual anchor name is a presentation detail and + ## cannot be queried by the user. + +proc `==`*(left, right: TagId): bool {.borrow.} +proc hash*(id: TagId): Hash {.borrow.} + +proc `==`*(left, right: AnchorId): bool {.borrow.} +proc `$`*(id: AnchorId): string {.borrow.} +proc hash*(id: AnchorId): Hash {.borrow.} diff --git a/private/dom.nim b/yaml/dom.nim similarity index 90% rename from private/dom.nim rename to yaml/dom.nim index ae8ce9e..83a57fa 100644 --- a/private/dom.nim +++ b/yaml/dom.nim @@ -4,6 +4,28 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. +import tables, streams +import common, stream, taglib, serialization, ../private/internal, parser, + presenter + +type + YamlNodeKind* = enum + yScalar, yMapping, ySequence + + YamlNode* = ref YamlNodeObj not nil + ## Represents a node in a ``YamlDocument``. + + YamlNodeObj* = object + tag*: string + case kind*: YamlNodeKind + of yScalar: content*: string + of ySequence: children*: seq[YamlNode] + of yMapping: pairs*: seq[tuple[key, value: YamlNode]] + + YamlDocument* = object + ## Represents a YAML document. + root*: YamlNode + proc newYamlNode*(content: string, tag: string = "?"): YamlNode = YamlNode(kind: yScalar, content: content, tag: tag) @@ -130,17 +152,17 @@ template processAnchoredEvent(target: untyped, c: SerializationContext): typed = else: target = yAnchorNone proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy): - YamlStream {.raises: [YamlStreamError].} = + YamlStream {.raises: [].} = var bys = newBufferYamlStream() c = newSerializationContext(a, proc(e: YamlStreamEvent) {.raises: [].} = - bys.buf.add(e) + bys.put(e) ) c.put(startDocEvent()) serializeNode(doc.root, c, a, tagLib) c.put(endDocEvent()) if a == asTidy: - for event in bys.buf.mitems(): + for event in bys.mitems(): case event.kind of yamlScalar: processAnchoredEvent(event.scalarAnchor, c) of yamlStartMap: processAnchoredEvent(event.mapAnchor, c) diff --git a/private/hints.nim b/yaml/hints.nim similarity index 77% rename from private/hints.nim rename to yaml/hints.nim index 71f0cdd..6359cf4 100644 --- a/private/hints.nim +++ b/yaml/hints.nim @@ -4,7 +4,34 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. +import macros +import ../private/internal + type + TypeHint* = enum + ## A type hint can be computed from scalar content and tells you what + ## NimYAML thinks the scalar's type is. It is generated by + ## `guessType <#guessType,string>`_ The first matching RegEx + ## in the following table will be the type hint of a scalar string. + ## + ## You can use it to determine the type of YAML scalars that have a '?' + ## non-specific tag, but using this feature is completely optional. + ## + ## ================== ========================= + ## Name RegEx + ## ================== ========================= + ## ``yTypeInteger`` ``0 | -? [1-9] [0-9]*`` + ## ``yTypeFloat`` ``-? [1-9] ( \. [0-9]* [1-9] )? ( e [-+] [1-9] [0-9]* )?`` + ## ``yTypeFloatInf`` ``-? \. (inf | Inf | INF)`` + ## ``yTypeFloatNaN`` ``-? \. (nan | NaN | NAN)`` + ## ``yTypeBoolTrue`` ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON`` + ## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF`` + ## ``yTypeNull`` ``~ | null | Null | NULL`` + ## ``yTypeUnknown`` ``*`` + ## ================== ========================= + yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue, + yTypeBoolFalse, yTypeNull, yTypeUnknown + YamlTypeHintState = enum ythInitial, ythF, ythFA, ythFAL, ythFALS, ythFALSE, @@ -150,7 +177,9 @@ template advanceTypeHint(ch: char) {.dirty.} = ythTR => ythTRU of 'y', 'Y': ythInitial => ythY -proc guessType*(scalar: string): TypeHint = +proc guessType*(scalar: string): TypeHint {.raises: [].} = + ## Parse scalar string according to the RegEx table documented at + ## `TypeHint <#TypeHind>`_. var typeHintState: YamlTypeHintState = ythInitial for c in scalar: advanceTypeHint(c) case typeHintState diff --git a/private/parse.nim b/yaml/parser.nim similarity index 89% rename from private/parse.nim rename to yaml/parser.nim index fda5574..fa66e3e 100644 --- a/private/parse.nim +++ b/yaml/parser.nim @@ -4,7 +4,29 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. +import tables, strutils, macros, streams +import common, taglib, stream, ../private/lex, ../private/internal + type + WarningCallback* = proc(line, column: int, lineContent: string, + message: string) + ## Callback for parser warnings. Currently, this callback may be called + ## on two occasions while parsing a YAML document stream: + ## + ## - If the version number in the ``%YAML`` directive does not match + ## ``1.2``. + ## - If there is an unknown directive encountered. + + YamlParser* = ref object + ## A parser object. Retains its ``TagLibrary`` across calls to + ## `parse <#parse,YamlParser,Stream>`_. Can be used + ## to access anchor names while parsing a YAML character stream, but + ## only until the document goes out of scope (i.e. until + ## ``yamlEndDocument`` is yielded). + tagLib: TagLibrary + callback: WarningCallback + anchors: Table[string, AnchorId] + FastParseLevelKind = enum fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey, fplSinglePairValue, fplDocument @@ -31,8 +53,51 @@ type LevelEndResult = enum lerNothing, lerOne, lerAdditionalMapEnd + YamlLoadingError* = object of Exception + ## Base class for all exceptions that may be raised during the process + ## of loading a YAML character stream. + line*: int ## line number (1-based) where the error was encountered + column*: int ## column number (1-based) where the error was encountered + lineContent*: string ## \ + ## content of the line where the error was encountered. Includes a + ## second line with a marker ``^`` at the position where the error + ## was encountered. + + YamlParserError* = object of YamlLoadingError + ## A parser error is raised if the character stream that is parsed is + ## not a valid YAML character stream. This stream cannot and will not be + ## parsed wholly nor partially and all events that have been emitted by + ## the YamlStream the parser provides should be discarded. + ## + ## A character stream is invalid YAML if and only if at least one of the + ## following conditions apply: + ## + ## - There are invalid characters in an element whose contents is + ## restricted to a limited set of characters. For example, there are + ## characters in a tag URI which are not valid URI characters. + ## - An element has invalid indentation. This can happen for example if + ## a block list element indicated by ``"- "`` is less indented than + ## the element in the previous line, but there is no block sequence + ## list open at the same indentation level. + ## - The YAML structure is invalid. For example, an explicit block map + ## indicated by ``"? "`` and ``": "`` may not suddenly have a block + ## sequence item (``"- "``) at the same indentation level. Another + ## possible violation is closing a flow style object with the wrong + ## closing character (``}``, ``]``) or not closing it at all. + ## - A custom tag shorthand is used that has not previously been + ## declared with a ``%TAG`` directive. + ## - Multiple tags or anchors are defined for the same node. + ## - An alias is used which does not map to any anchor that has + ## previously been declared in the same document. + ## - An alias has a tag or anchor associated with it. + ## + ## Some elements in this list are vague. For a detailed description of a + ## valid YAML character stream, see the YAML specification. + proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(), callback: WarningCallback = nil): YamlParser = + ## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called + ## whenever the parser yields a warning. new(result) result.tagLib = tagLib result.callback = callback @@ -959,18 +1024,18 @@ proc lastTokenContext(s: YamlStream, line, column: var int, # --- parser initialization --- proc init(c: ParserContext, p: YamlParser) = + c.basicInit(lastTokenContext) c.p = p c.ancestry = newSeq[FastParseLevel]() c.initDocValues() c.flowdepth = 0 - c.isFinished = false - c.peeked = false c.nextImpl = stateInitial c.explicitFlowKey = false - c.lastTokenContextImpl = lastTokenContext c.advance() -proc parse*(p: YamlParser, s: Stream): YamlStream = +proc parse*(p: YamlParser, s: Stream): YamlStream + {.raises: [YamlParserError].} = + ## Parse the given stream as YAML character stream. let c = new(ParserContext) try: c.lex = newYamlLexer(s) except: @@ -984,7 +1049,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream = c.init(p) result = c -proc parse*(p: YamlParser, str: string): YamlStream = +proc parse*(p: YamlParser, str: string): YamlStream + {.raises: [YamlParserError].} = + ## Parse the given string as YAML character stream. let c = new(ParserContext) c.lex = newYamlLexer(str) c.init(p) diff --git a/private/presenter.nim b/yaml/presenter.nim similarity index 82% rename from private/presenter.nim rename to yaml/presenter.nim index 1fee0e8..ec84568 100644 --- a/private/presenter.nim +++ b/yaml/presenter.nim @@ -4,7 +4,95 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. +import streams, queues, strutils +import common, taglib, stream, ../private/internal, hints, parser, stream + type + PresentationStyle* = enum + ## Different styles for YAML character stream output. + ## + ## - ``ypsMinimal``: Single-line flow-only output which tries to + ## use as few characters as possible. + ## - ``ypsCanonical``: Canonical YAML output. Writes all tags except + ## for the non-specific tags ``?`` and ``!``, uses flow style, quotes + ## all string scalars. + ## - ``ypsDefault``: Tries to be as human-readable as possible. Uses + ## block style by default, but tries to condense mappings and + ## sequences which only contain scalar nodes into a single line using + ## flow style. + ## - ``ypsJson``: Omits the ``%YAML`` directive and the ``---`` + ## marker. Uses flow style. Flattens anchors and aliases, omits tags. + ## Output will be parseable as JSON. ``YamlStream`` to dump may only + ## contain one document. + ## - ``ypsBlockOnly``: Formats all output in block style, does not use + ## flow style at all. + psMinimal, psCanonical, psDefault, psJson, psBlockOnly + + TagStyle* = enum + ## Whether object should be serialized with explicit tags. + ## + ## - ``tsNone``: No tags will be outputted unless necessary. + ## - ``tsRootOnly``: A tag will only be outputted for the root tag and + ## where necessary. + ## - ``tsAll``: Tags will be outputted for every object. + tsNone, tsRootOnly, tsAll + + AnchorStyle* = enum + ## How ref object should be serialized. + ## + ## - ``asNone``: No anchors will be outputted. Values present at + ## multiple places in the content that should be serialized will be + ## fully serialized at every occurence. If the content is cyclic, this + ## will lead to an endless loop! + ## - ``asTidy``: Anchors will only be generated for objects that + ## actually occur more than once in the content to be serialized. + ## This is a bit slower and needs more memory than ``asAlways``. + ## - ``asAlways``: Achors will be generated for every ref object in the + ## content to be serialized, regardless of whether the object is + ## referenced again afterwards + asNone, asTidy, asAlways + + NewLineStyle* = enum + ## What kind of newline sequence is used when presenting. + ## + ## - ``nlLF``: Use a single linefeed char as newline. + ## - ``nlCRLF``: Use a sequence of carriage return and linefeed as + ## newline. + ## - ``nlOSDefault``: Use the target operation system's default newline + ## sequence (CRLF on Windows, LF everywhere else). + nlLF, nlCRLF, nlOSDefault + + OutputYamlVersion* = enum + ## Specify which YAML version number the presenter shall emit. The + ## presenter will always emit content that is valid YAML 1.1, but by + ## default will write a directive ``%YAML 1.2``. For compatibility with + ## other YAML implementations, it is possible to change this here. + ## + ## It is also possible to specify that the presenter shall not emit any + ## YAML version. The generated content is then guaranteed to be valid + ## YAML 1.1 and 1.2 (but not 1.0 or any newer YAML version). + ov1_2, ov1_1, ovNone + + PresentationOptions* = object + ## Options for generating a YAML character stream + style*: PresentationStyle + indentationStep*: int + newlines*: NewLineStyle + outputVersion*: OutputYamlVersion + + YamlPresenterJsonError* = object of Exception + ## Exception that may be raised by the YAML presenter when it is + ## instructed to output JSON, but is unable to do so. This may occur if: + ## + ## - The given `YamlStream <#YamlStream>`_ contains a map which has any + ## non-scalar type as key. + ## - Any float scalar bears a ``NaN`` or positive/negative infinity value + + YamlPresenterOutputError* = object of Exception + ## Exception that may be raised by the YAML presenter. This occurs if + ## writing character data to the output stream raises any exception. + ## The error that has occurred is available from ``parent``. + DumperState = enum dBlockExplicitMapKey, dBlockImplicitMapKey, dBlockMapValue, dBlockInlineMap, dBlockSequenceItem, dFlowImplicitMapKey, dFlowMapValue, @@ -15,11 +103,18 @@ type PresenterTarget = Stream | ptr[string] +const + defaultPresentationOptions* = + PresentationOptions(style: psDefault, indentationStep: 2, + newlines: nlOSDefault) + proc defineOptions*(style: PresentationStyle = psDefault, indentationStep: int = 2, newlines: NewLineStyle = nlOSDefault, outputVersion: OutputYamlVersion = ov1_2): - PresentationOptions = + PresentationOptions {.raises: [].} = + ## Define a set of options for presentation. Convenience proc that requires + ## you to only set those values that should not equal the default. PresentationOptions(style: style, indentationStep: indentationStep, newlines: newlines, outputVersion: outputVersion) @@ -607,12 +702,17 @@ proc doPresent(s: var YamlStream, target: PresenterTarget, proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, - options: PresentationOptions = defaultPresentationOptions) = + options: PresentationOptions = defaultPresentationOptions) + {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} = + ## Convert ``s`` to a YAML character stream and write it to ``target``. doPresent(s, target, tagLib, options) proc present*(s: var YamlStream, tagLib: TagLibrary, options: PresentationOptions = defaultPresentationOptions): - string = + string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} = + ## Convert ``s`` to a YAML character stream and return it as string. result = "" doPresent(s, addr result, tagLib, options) @@ -647,7 +747,7 @@ proc doTransform(input: Stream | string, output: PresenterTarget, of yTypeUnknown: event.scalarTag = yTagString elif event.scalarTag == yTagExclamationMark: event.scalarTag = yTagString - BufferYamlStream(bys).buf.add(e) + BufferYamlStream(bys).put(e) present(bys, output, tagLib, options) else: present(events, output, tagLib, options) except YamlStreamError: @@ -658,11 +758,20 @@ proc doTransform(input: Stream | string, output: PresenterTarget, else: internalError("Unexpected exception: " & e.parent.repr) proc transform*(input: Stream | string, output: Stream, - options: PresentationOptions = defaultPresentationOptions) = + options: PresentationOptions = defaultPresentationOptions) + {.raises: [IOError, YamlParserError, YamlPresenterJsonError, + YamlPresenterOutputError].} = + ## Parser ``input`` as YAML character stream and then dump it to ``output`` + ## while resolving non-specific tags to the ones in the YAML core tag + ## library. doTransform(input, output, options) proc transform*(input: Stream | string, options: PresentationOptions = defaultPresentationOptions): - string = + string {.raises: [IOError, YamlParserError, YamlPresenterJsonError, + YamlPresenterOutputError].} = + ## Parser ``input`` as YAML character stream, resolves non-specific tags to + ## the ones in the YAML core tag library, and then returns a serialized + ## YAML string that represents the stream. result = "" doTransform(input, addr result, options) diff --git a/private/serialization.nim b/yaml/serialization.nim similarity index 89% rename from private/serialization.nim rename to yaml/serialization.nim index 251422b..beac9d3 100644 --- a/private/serialization.nim +++ b/yaml/serialization.nim @@ -4,6 +4,72 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. +import tables, typetraits, strutils, macros, streams +import parser, common, taglib, presenter, stream, ../private/internal, hints + +type + SerializationContext* = ref object + ## Context information for the process of serializing YAML from Nim values. + refs*: Table[pointer, AnchorId] + style: AnchorStyle + nextAnchorId*: AnchorId + put*: proc(e: YamlStreamEvent) {.raises: [], closure.} + + ConstructionContext* = ref object + ## Context information for the process of constructing Nim values from YAML. + refs*: Table[AnchorId, pointer] + + YamlConstructionError* = object of YamlLoadingError + ## Exception that may be raised when constructing data objects from a + ## `YamlStream <#YamlStream>`_. The fields ``line``, ``column`` and + ## ``lineContent`` are only available if the costructing proc also does + ## parsing, because otherwise this information is not available to the + ## costruction proc. + +# forward declares + +proc constructChild*[T](s: var YamlStream, c: ConstructionContext, + result: var T) + {.raises: [YamlConstructionError, YamlStreamError].} + ## Constructs an arbitrary Nim value from a part of a YAML stream. + ## The stream will advance until after the finishing token that was used + ## for constructing the value. The ``ConstructionContext`` is needed for + ## potential child objects which may be refs. + +proc constructChild*(s: var YamlStream, c: ConstructionContext, + result: var string) + {.raises: [YamlConstructionError, YamlStreamError].} + ## Constructs a Nim value that is a string from a part of a YAML stream. + ## This specialization takes care of possible nil strings. + +proc constructChild*[T](s: var YamlStream, c: ConstructionContext, + result: var seq[T]) + {.raises: [YamlConstructionError, YamlStreamError].} + ## Constructs a Nim value that is a string from a part of a YAML stream. + ## This specialization takes care of possible nil seqs. + +proc constructChild*[O](s: var YamlStream, c: ConstructionContext, + result: var ref O) + {.raises: [YamlStreamError].} + ## Constructs an arbitrary Nim value from a part of a YAML stream. + ## The stream will advance until after the finishing token that was used + ## for constructing the value. The object may be constructed from an alias + ## node which will be resolved using the ``ConstructionContext``. + +proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext) + {.raises: [YamlStreamError].} + ## Represents an arbitrary Nim reference value as YAML object. The object + ## may be represented as alias node if it is already present in the + ## ``SerializationContext``. + +proc representChild*(value: string, ts: TagStyle, c: SerializationContext) + {.inline, raises: [].} + ## Represents a Nim string. Supports nil strings. + +proc representChild*[O](value: O, ts: TagStyle, + c: SerializationContext) {.raises: [YamlStreamError].} + ## Represents an arbitrary Nim object as YAML object. + proc newConstructionContext*(): ConstructionContext = new(result) result.refs = initTable[AnchorId, pointer]() @@ -118,26 +184,16 @@ proc representObject*(value: int, tagStyle: TagStyle, e.parent = getCurrentException() raise e -{.push overflowChecks: on.} -proc parseBiggestUInt(s: string): uint64 = - result = 0 - if s[0] == '0' and s[1] in {'x', 'X'}: - result = parseHex[uint64](s) - elif s[0] == '0' and s[1] in {'o', 'O'}: - result = parseOctal[uint64](s) - else: - for c in s: - if c in {'0'..'9'}: result = result * 10.uint64 + (uint64(c) - uint64('0')) - elif c == '_': discard - else: raise newException(ValueError, "Invalid char in uint: " & c) -{.pop.} - proc constructObject*[T: uint8|uint16|uint32|uint64]( s: var YamlStream, c: ConstructionContext, result: var T) {.raises: [YamlConstructionError, YamlStreamError].} = ## construct an unsigned integer value from a YAML scalar constructScalarItem(s, item, T): - result = T(yaml.parseBiggestUInt(item.scalarContent)) + if item.scalarContent[0] == '0' and item.scalarContent[1] in {'x', 'X'}: + result = parseHex[T](item.scalarContent) + elif item.scalarContent[0] == '0' and item.scalarContent[1] in {'o', 'O'}: + result = parseOctal[T](item.scalarContent) + else: result = T(parseBiggestUInt(item.scalarContent)) proc constructObject*(s: var YamlStream, c: ConstructionContext, result: var uint) @@ -763,7 +819,9 @@ proc representChild*[O](value: O, ts: TagStyle, representObject(value, ts, c, if ts == tsNone: yTagQuestionMark else: yamlTag(O)) -proc construct*[T](s: var YamlStream, target: var T) = +proc construct*[T](s: var YamlStream, target: var T) + {.raises: [YamlStreamError].} = + ## Constructs a Nim value from a YAML stream. var context = newConstructionContext() try: var e = s.next() @@ -785,7 +843,9 @@ proc construct*[T](s: var YamlStream, target: var T) = ex.parent = getCurrentException() raise ex -proc load*[K](input: Stream | string, target: var K) = +proc load*[K](input: Stream | string, target: var K) + {.raises: [YamlConstructionError, IOError, YamlParserError].} = + ## Loads a Nim value from a YAML character stream. var parser = newYamlParser(serializationTagLibrary) events = parser.parse(input) @@ -826,16 +886,18 @@ proc setAnchor(a: var AnchorId, q: var Table[pointer, AnchorId]) if a != yAnchorNone: a = q.getOrDefault(cast[pointer](a)) proc represent*[T](value: T, ts: TagStyle = tsRootOnly, - a: AnchorStyle = asTidy): YamlStream = + a: AnchorStyle = asTidy): YamlStream + {.raises: [YamlStreamError].} = + ## Represents a Nim value as ``YamlStream`` var bys = newBufferYamlStream() var context = newSerializationContext(a, proc(e: YamlStreamEvent) = - bys.buf.add(e) + bys.put(e) ) - bys.buf.add(startDocEvent()) + bys.put(startDocEvent()) representChild(value, ts, context) - bys.buf.add(endDocEvent()) + bys.put(endDocEvent()) if a == asTidy: - for item in bys.buf.mitems(): + for item in bys.mitems(): case item.kind of yamlStartMap: item.mapAnchor.setAnchor(context.refs) of yamlStartSeq: item.seqAnchor.setAnchor(context.refs) @@ -845,7 +907,10 @@ proc represent*[T](value: T, ts: TagStyle = tsRootOnly, proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly, anchorStyle: AnchorStyle = asTidy, - options: PresentationOptions = defaultPresentationOptions) = + options: PresentationOptions = defaultPresentationOptions) + {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} = + ## Dump a Nim value as YAML character stream. var events = represent(value, if options.style == psCanonical: tsAll else: tagStyle, if options.style == psJson: asNone else: anchorStyle) @@ -856,7 +921,9 @@ proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly, proc dump*[K](value: K, tagStyle: TagStyle = tsRootOnly, anchorStyle: AnchorStyle = asTidy, options: PresentationOptions = defaultPresentationOptions): - string = + string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} = + ## Dump a Nim value as YAML into a string var events = represent(value, if options.style == psCanonical: tsAll else: tagStyle, if options.style == psJson: asNone else: anchorStyle) diff --git a/yaml/stream.nim b/yaml/stream.nim new file mode 100644 index 0000000..d9f5335 --- /dev/null +++ b/yaml/stream.nim @@ -0,0 +1,281 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2015 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +import common, ../private/internal +import taglib + +type + YamlStreamEventKind* = enum + ## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds + ## are discussed in `YamlStreamEvent <#YamlStreamEvent>`_. + yamlStartDoc, yamlEndDoc, yamlStartMap, yamlEndMap, + yamlStartSeq, yamlEndSeq, yamlScalar, yamlAlias + + YamlStreamEvent* = object + ## An element from a `YamlStream <#YamlStream>`_. Events that start an + ## object (``yamlStartMap``, ``yamlStartSeq``, ``yamlScalar``) have + ## an optional anchor and a tag associated with them. The anchor will be + ## set to ``yAnchorNone`` if it doesn't exist. + ## + ## A non-existing tag in the YAML character stream will be resolved to + ## the non-specific tags ``?`` or ``!`` according to the YAML + ## specification. These are by convention mapped to the ``TagId`` s + ## ``yTagQuestionMark`` and ``yTagExclamationMark`` respectively. + ## Mapping is done by a `TagLibrary <#TagLibrary>`_. + case kind*: YamlStreamEventKind + of yamlStartMap: + mapAnchor* : AnchorId + mapTag* : TagId + of yamlStartSeq: + seqAnchor* : AnchorId + seqTag* : TagId + of yamlScalar: + scalarAnchor* : AnchorId + scalarTag* : TagId + scalarContent*: string # may not be nil (but empty) + of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard + of yamlAlias: + aliasTarget* : AnchorId + + YamlStream* = ref object of RootObj ## \ + ## A ``YamlStream`` is an iterator-like object that yields a + ## well-formed stream of ``YamlStreamEvents``. Well-formed means that + ## every ``yamlStartMap`` is terminated by a ``yamlEndMap``, every + ## ``yamlStartSeq`` is terminated by a ``yamlEndSeq`` and every + ## ``yamlStartDoc`` is terminated by a ``yamlEndDoc``. Moreover, every + ## emitted mapping has an even number of children. + ## + ## The creator of a ``YamlStream`` is responsible for it being + ## well-formed. A user of the stream may assume that it is well-formed + ## and is not required to check for it. The procs in this module will + ## always yield a well-formed ``YamlStream`` and expect it to be + ## well-formed if they take it as input parameter. + nextImpl*: proc(s: YamlStream, e: var YamlStreamEvent): bool + lastTokenContextImpl*: + proc(s: YamlStream, line, column: var int, + lineContent: var string): bool {.raises: [].} + isFinished*: bool + peeked: bool + cached: YamlStreamEvent + + YamlStreamError* = object of Exception + ## Exception that may be raised by a ``YamlStream`` when the underlying + ## backend raises an exception. The error that has occurred is + ## available from ``parent``. + +proc noLastContext(s: YamlStream, line, column: var int, + lineContent: var string): bool = + (line, column, lineContent) = (-1, -1, "") + result = false + +proc basicInit*(s: YamlStream, lastTokenContextImpl: + proc(s: YamlStream, line, column: var int, lineContent: var string): bool + {.raises: [].} = noLastContext) + {.raises: [].} = + ## initialize basic values of the YamlStream. Call this in your constructor + ## if you subclass YamlStream. + s.peeked = false + s.isFinished = false + s.lastTokenContextImpl = lastTokenContextImpl + +when not defined(JS): + type IteratorYamlStream = ref object of YamlStream + backend: iterator(): YamlStreamEvent + + proc initYamlStream*(backend: iterator(): YamlStreamEvent): YamlStream + {.raises: [].} = + ## Creates a new ``YamlStream`` that uses the given iterator as backend. + result = new(IteratorYamlStream) + result.basicInit() + IteratorYamlStream(result).backend = backend + result.nextImpl = proc(s: YamlStream, e: var YamlStreamEvent): bool = + e = IteratorYamlStream(s).backend() + if finished(IteratorYamlStream(s).backend): + s.isFinished = true + result = false + else: result = true + +type + BufferYamlStream* = ref object of YamlStream + pos: int + buf: seq[YamlStreamEvent] not nil + +proc newBufferYamlStream*(): BufferYamlStream not nil = + result = cast[BufferYamlStream not nil](new(BufferYamlStream)) + result.basicInit() + result.buf = @[] + result.pos = 0 + result.nextImpl = proc(s: YamlStream, e: var YamlStreamEvent): bool = + let bys = BufferYamlStream(s) + if bys.pos == bys.buf.len: + result = false + s.isFinished = true + else: + e = bys.buf[bys.pos] + inc(bys.pos) + result = true + +proc put*(bys: BufferYamlStream, e: YamlStreamEvent) {.raises: [].} = + bys.buf.add(e) + +proc next*(s: YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} = + ## Get the next item of the stream. Requires ``finished(s) == true``. + ## If the backend yields an exception, that exception will be encapsulated + ## into a ``YamlStreamError``, which will be raised. + if s.peeked: + s.peeked = false + shallowCopy(result, s.cached) + return + else: + yAssert(not s.isFinished) + try: + while true: + if s.nextImpl(s, result): break + yAssert(not s.isFinished) + except YamlStreamError: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur.parent + raise e + except Exception: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur + raise e + +proc peek*(s: YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} = + ## Get the next item of the stream without advancing the stream. + ## Requires ``finished(s) == true``. Handles exceptions of the backend like + ## ``next()``. + if not s.peeked: + shallowCopy(s.cached, s.next()) + s.peeked = true + shallowCopy(result, s.cached) + +proc `peek=`*(s: YamlStream, value: YamlStreamEvent) {.raises: [].} = + ## Set the next item of the stream. Will replace a previously peeked item, + ## if one exists. + s.cached = value + s.peeked = true + +proc finished*(s: YamlStream): bool {.raises: [YamlStreamError].} = + ## ``true`` if no more items are available in the stream. Handles exceptions + ## of the backend like ``next()``. + if s.peeked: result = false + else: + try: + while true: + if s.isFinished: return true + if s.nextImpl(s, s.cached): + s.peeked = true + return false + except YamlStreamError: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur.parent + raise e + except Exception: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur + raise e + +proc getLastTokenContext*(s: YamlStream, line, column: var int, + lineContent: var string): bool = + ## ``true`` if source context information is available about the last returned + ## token. If ``true``, line, column and lineContent are set to position and + ## line content where the last token has been read from. + result = s.lastTokenContextImpl(s, line, column, lineContent) + +iterator items*(s: YamlStream): YamlStreamEvent + {.raises: [YamlStreamError].} = + ## Iterate over all items of the stream. You may not use ``peek()`` on the + ## stream while iterating. + while not s.finished(): yield s.next() + +iterator mitems*(bys: BufferYamlStream): var YamlStreamEvent {.raises: [].} = + ## Iterate over all items of the stream. You may not use ``peek()`` on the + ## stream while iterating. + for e in bys.buf.mitems(): yield e + +proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool {.raises: [].} = + ## compares all existing fields of the given items + if left.kind != right.kind: return false + case left.kind + of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlEndSeq: result = true + of yamlStartMap: + result = left.mapAnchor == right.mapAnchor and left.mapTag == right.mapTag + of yamlStartSeq: + result = left.seqAnchor == right.seqAnchor and left.seqTag == right.seqTag + of yamlScalar: + result = left.scalarAnchor == right.scalarAnchor and + left.scalarTag == right.scalarTag and + left.scalarContent == right.scalarContent + of yamlAlias: result = left.aliasTarget == right.aliasTarget + +proc `$`*(event: YamlStreamEvent): string {.raises: [].} = + ## outputs a human-readable string describing the given event + result = $event.kind & '(' + case event.kind + of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard + of yamlStartMap: + result &= "tag=" & $event.mapTag + if event.mapAnchor != yAnchorNone: result &= ", anchor=" & $event.mapAnchor + of yamlStartSeq: + result &= "tag=" & $event.seqTag + if event.seqAnchor != yAnchorNone: result &= ", anchor=" & $event.seqAnchor + of yamlScalar: + result &= "tag=" & $event.scalarTag + if event.scalarAnchor != yAnchorNone: + result &= ", anchor=" & $event.scalarAnchor + result &= ", content=\"" & event.scalarContent & '\"' + of yamlAlias: + result &= "aliasTarget=" & $event.aliasTarget + result &= ")" + +proc tag*(event: YamlStreamEvent): TagId {.raises: [FieldError].} = + ## returns the tag of the given event + case event.kind + of yamlStartMap: result = event.mapTag + of yamlStartSeq: result = event.seqTag + of yamlScalar: result = event.scalarTag + else: raise newException(FieldError, "Event " & $event.kind & " has no tag") + +proc startDocEvent*(): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the start of a YAML document + result = YamlStreamEvent(kind: yamlStartDoc) + +proc endDocEvent*(): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the end of a YAML document + result = YamlStreamEvent(kind: yamlEndDoc) + +proc startMapEvent*(tag: TagId = yTagQuestionMark, + anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the start of a YAML mapping + result = YamlStreamEvent(kind: yamlStartMap, mapTag: tag, mapAnchor: anchor) + +proc endMapEvent*(): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the end of a YAML mapping + result = YamlStreamEvent(kind: yamlEndMap) + +proc startSeqEvent*(tag: TagId = yTagQuestionMark, + anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the beginning of a YAML sequence + result = YamlStreamEvent(kind: yamlStartSeq, seqTag: tag, seqAnchor: anchor) + +proc endSeqEvent*(): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the end of a YAML sequence + result = YamlStreamEvent(kind: yamlEndSeq) + +proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, + anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that represents a YAML scalar + result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag, + scalarAnchor: anchor, scalarContent: content) + +proc aliasEvent*(anchor: AnchorId): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that represents a YAML alias + result = YamlStreamEvent(kind: yamlAlias, aliasTarget: anchor) diff --git a/yaml/taglib.nim b/yaml/taglib.nim new file mode 100644 index 0000000..d722e90 --- /dev/null +++ b/yaml/taglib.nim @@ -0,0 +1,290 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2015 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +import tables, macros +import common + +type + TagLibrary* = ref object + ## A ``TagLibrary`` maps tag URIs to ``TagId`` s. + ## + ## When `YamlParser <#YamlParser>`_ encounters tags not existing in the + ## tag library, it will use + ## `registerUri <#registerUri,TagLibrary,string>`_ to add + ## the tag to the library. + ## + ## You can base your tag library on common tag libraries by initializing + ## them with `initFailsafeTagLibrary <#initFailsafeTagLibrary>`_, + ## `initCoreTagLibrary <#initCoreTagLibrary>`_ or + ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_. + tags*: Table[string, TagId] + nextCustomTagId*: TagId + secondaryPrefix*: string + +const + # failsafe schema + + yTagExclamationMark*: TagId = 0.TagId ## ``!`` non-specific tag + yTagQuestionMark* : TagId = 1.TagId ## ``?`` non-specific tag + yTagString* : TagId = 2.TagId ## \ + ## `!!str `_ tag + yTagSequence* : TagId = 3.TagId ## \ + ## `!!seq `_ tag + yTagMapping* : TagId = 4.TagId ## \ + ## `!!map `_ tag + + # json & core schema + + yTagNull* : TagId = 5.TagId ## \ + ## `!!null `_ tag + yTagBoolean* : TagId = 6.TagId ## \ + ## `!!bool `_ tag + yTagInteger* : TagId = 7.TagId ## \ + ## `!!int `_ tag + yTagFloat* : TagId = 8.TagId ## \ + ## `!!float `_ tag + + # other language-independent YAML types (from http://yaml.org/type/ ) + + yTagOrderedMap* : TagId = 9.TagId ## \ + ## `!!omap `_ tag + yTagPairs* : TagId = 10.TagId ## \ + ## `!!pairs `_ tag + yTagSet* : TagId = 11.TagId ## \ + ## `!!set `_ tag + yTagBinary* : TagId = 12.TagId ## \ + ## `!!binary `_ tag + yTagMerge* : TagId = 13.TagId ## \ + ## `!!merge `_ tag + yTagTimestamp* : TagId = 14.TagId ## \ + ## `!!timestamp `_ tag + yTagValue* : TagId = 15.TagId ## \ + ## `!!value `_ tag + yTagYaml* : TagId = 16.TagId ## \ + ## `!!yaml `_ tag + + yTagNimField* : TagId = 100.TagId ## \ + ## This tag is used in serialization for the name of a field of an + ## object. It may contain any string scalar that is a valid Nim symbol. + + yTagNimNilString* : TagId = 101.TagId ## for strings that are nil + yTagNimNilSeq* : TagId = 102.TagId ## \ + ## for seqs that are nil. This tag is used regardless of the seq's generic + ## type parameter. + + yFirstCustomTagId* : TagId = 1000.TagId ## \ + ## The first ``TagId`` which should be assigned to an URI that does not + ## exist in the ``YamlTagLibrary`` which is used for parsing. + + yAnchorNone*: AnchorId = (-1).AnchorId ## \ + ## yielded when no anchor was defined for a YAML node + + yamlTagRepositoryPrefix* = "tag:yaml.org,2002:" + +proc `$`*(id: TagId): string {.raises: [].} = + case id + of yTagQuestionMark: "?" + of yTagExclamationMark: "!" + of yTagString: "!!str" + of yTagSequence: "!!seq" + of yTagMapping: "!!map" + of yTagNull: "!!null" + of yTagBoolean: "!!bool" + of yTagInteger: "!!int" + of yTagFloat: "!!float" + of yTagOrderedMap: "!!omap" + of yTagPairs: "!!pairs" + of yTagSet: "!!set" + of yTagBinary: "!!binary" + of yTagMerge: "!!merge" + of yTagTimestamp: "!!timestamp" + of yTagValue: "!!value" + of yTagYaml: "!!yaml" + of yTagNimField: "!nim:field" + else: "<" & $int(id) & ">" + +proc initTagLibrary*(): TagLibrary {.raises: [].} = + ## initializes the ``tags`` table and sets ``nextCustomTagId`` to + ## ``yFirstCustomTagId``. + new(result) + result.tags = initTable[string, TagId]() + result.secondaryPrefix = yamlTagRepositoryPrefix + result.nextCustomTagId = yFirstCustomTagId + +proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} = + ## registers a custom tag URI with a ``TagLibrary``. The URI will get + ## the ``TagId`` ``nextCustomTagId``, which will be incremented. + tagLib.tags[uri] = tagLib.nextCustomTagId + result = tagLib.nextCustomTagId + tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1) + +proc uri*(tagLib: TagLibrary, id: TagId): string {.raises: [KeyError].} = + ## retrieve the URI a ``TagId`` maps to. + for iUri, iId in tagLib.tags.pairs: + if iId == id: return iUri + raise newException(KeyError, "Unknown tag id: " & $id) + +proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} = + ## Contains only: + ## - ``!`` + ## - ``?`` + ## - ``!!str`` + ## - ``!!map`` + ## - ``!!seq`` + result = initTagLibrary() + result.tags["!"] = yTagExclamationMark + result.tags["?"] = yTagQuestionMark + result.tags["tag:yaml.org,2002:str"] = yTagString + result.tags["tag:yaml.org,2002:seq"] = yTagSequence + result.tags["tag:yaml.org,2002:map"] = yTagMapping + +proc initCoreTagLibrary*(): TagLibrary {.raises: [].} = + ## Contains everything in ``initFailsafeTagLibrary`` plus: + ## - ``!!null`` + ## - ``!!bool`` + ## - ``!!int`` + ## - ``!!float`` + result = initFailsafeTagLibrary() + result.tags["tag:yaml.org,2002:null"] = yTagNull + result.tags["tag:yaml.org,2002:bool"] = yTagBoolean + result.tags["tag:yaml.org,2002:int"] = yTagInteger + result.tags["tag:yaml.org,2002:float"] = yTagFloat + +proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} = + ## Contains everything from ``initCoreTagLibrary`` plus: + ## - ``!!omap`` + ## - ``!!pairs`` + ## - ``!!set`` + ## - ``!!binary`` + ## - ``!!merge`` + ## - ``!!timestamp`` + ## - ``!!value`` + ## - ``!!yaml`` + result = initCoreTagLibrary() + result.tags["tag:yaml.org,2002:omap"] = yTagOrderedMap + result.tags["tag:yaml.org,2002:pairs"] = yTagPairs + result.tags["tag:yaml.org,2002:binary"] = yTagBinary + result.tags["tag:yaml.org,2002:merge"] = yTagMerge + result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp + result.tags["tag:yaml.org,2002:value"] = yTagValue + result.tags["tag:yaml.org,2002:yaml"] = yTagYaml + + +proc initSerializationTagLibrary*(): TagLibrary = + result = initTagLibrary() + result.tags["!"] = yTagExclamationMark + result.tags["?"] = yTagQuestionMark + result.tags["tag:yaml.org,2002:str"] = yTagString + result.tags["tag:yaml.org,2002:null"] = yTagNull + result.tags["tag:yaml.org,2002:bool"] = yTagBoolean + result.tags["tag:yaml.org,2002:float"] = yTagFloat + result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp + result.tags["tag:yaml.org,2002:value"] = yTagValue + result.tags["tag:yaml.org,2002:binary"] = yTagBinary + result.tags["!nim:field"] = yTagNimField + result.tags["!nim:nil:string"] = yTagNimNilString + result.tags["!nim:nil:seq"] = yTagNimNilSeq + +var + serializationTagLibrary* = initSerializationTagLibrary() ## \ + ## contains all local tags that are used for type serialization. Does + ## not contain any of the specific default tags for sequences or maps, + ## as those are not suited for Nim's static type system. + ## + ## Should not be modified manually. Will be extended by + ## `serializable <#serializable,stmt,stmt>`_. + +var + nextStaticTagId {.compileTime.} = 100.TagId ## \ + ## used for generating unique TagIds with ``setTagUri``. + registeredUris {.compileTime.} = newSeq[string]() ## \ + ## Since Table doesn't really work at compile time, we also store + ## registered URIs here to be able to generate a static compiler error + ## when the user tries to register an URI more than once. + +template setTagUri*(t: typedesc, uri: string): typed = + ## Associate the given uri with a certain type. This uri is used as YAML tag + ## when loading and dumping values of this type. + when uri in registeredUris: + {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .} + const id {.genSym.} = nextStaticTagId + static: + registeredUris.add(uri) + nextStaticTagId = TagId(int(nextStaticTagId) + 1) + when nextStaticTagId == yFirstCustomTagId: + {.fatal: "Too many tags!".} + serializationTagLibrary.tags[uri] = id + proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = id + ## autogenerated + +template setTagUri*(t: typedesc, uri: string, idName: untyped): typed = + ## Like `setTagUri <#setTagUri.t,typedesc,string>`_, but lets + ## you choose a symbol for the `TagId <#TagId>`_ of the uri. This is only + ## necessary if you want to implement serialization / construction yourself. + when uri in registeredUris: + {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .} + const idName* = nextStaticTagId + static: + registeredUris.add(uri) + nextStaticTagId = TagId(int(nextStaticTagId) + 1) + serializationTagLibrary.tags[uri] = idName + proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName + ## autogenerated + +proc canBeImplicit(t: typedesc): bool {.compileTime.} = + let tDesc = getType(t) + if tDesc.kind != nnkObjectTy: return false + if tDesc[2].len != 1: return false + if tDesc[2][0].kind != nnkRecCase: return false + var foundEmptyBranch = false + for i in 1.. tDesc[2][0].len - 1: + case tDesc[2][0][i][1].len # branch contents + of 0: + if foundEmptyBranch: return false + else: foundEmptyBranch = true + of 1: discard + else: return false + return true + +template markAsImplicit*(t: typedesc): typed = + ## Mark a variant object type as implicit. This requires the type to consist + ## of nothing but a case expression and each branch of the case expression + ## containing exactly one field - with the exception that one branch may + ## contain zero fields. + when canBeImplicit(t): + # this will be checked by means of compiles(implicitVariantObject(...)) + proc implicitVariantObject*(unused: t) = discard + else: + {. fatal: "This type cannot be marked as implicit" .} + +static: + # standard YAML tags used by serialization + registeredUris.add("!") + registeredUris.add("?") + registeredUris.add("tag:yaml.org,2002:str") + registeredUris.add("tag:yaml.org,2002:null") + registeredUris.add("tag:yaml.org,2002:bool") + registeredUris.add("tag:yaml.org,2002:float") + registeredUris.add("tag:yaml.org,2002:timestamp") + registeredUris.add("tag:yaml.org,2002:value") + registeredUris.add("tag:yaml.org,2002:binary") + # special tags used by serialization + registeredUris.add("!nim:field") + registeredUris.add("!nim:nil:string") + registeredUris.add("!nim:nil:seq") + +# tags for Nim's standard types +setTagUri(char, "!nim:system:char", yTagNimChar) +setTagUri(int8, "!nim:system:int8", yTagNimInt8) +setTagUri(int16, "!nim:system:int16", yTagNimInt16) +setTagUri(int32, "!nim:system:int32", yTagNimInt32) +setTagUri(int64, "!nim:system:int64", yTagNimInt64) +setTagUri(uint8, "!nim:system:uint8", yTagNimUInt8) +setTagUri(uint16, "!nim:system:uint16", yTagNimUInt16) +setTagUri(uint32, "!nim:system:uint32", yTagNimUInt32) +setTagUri(uint64, "!nim:system:uint64", yTagNimUInt64) +setTagUri(float32, "!nim:system:float32", yTagNimFloat32) +setTagUri(float64, "!nim:system:float64", yTagNimFloat64) \ No newline at end of file diff --git a/private/json.nim b/yaml/tojson.nim similarity index 83% rename from private/json.nim rename to yaml/tojson.nim index 443ad7e..e29427a 100644 --- a/private/json.nim +++ b/yaml/tojson.nim @@ -4,6 +4,9 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. +import json, streams, strutils, tables +import common, taglib, hints, serialization, stream, ../private/internal, parser + type Level = tuple[node: JsonNode, key: string] proc initLevel(node: JsonNode): Level {.raises: [].} = @@ -66,7 +69,20 @@ proc jsonFromScalar(content: string, tag: TagId): JsonNode e.parent = getCurrentException() raise e -proc constructJson*(s: var YamlStream): seq[JsonNode] = +proc constructJson*(s: var YamlStream): seq[JsonNode] + {.raises: [YamlConstructionError, YamlStreamError].} = + ## Construct an in-memory JSON tree from a YAML event stream. The stream may + ## not contain any tags apart from those in ``coreTagLibrary``. Anchors and + ## aliases will be resolved. Maps in the input must not contain + ## non-scalars as keys. Each element of the result represents one document + ## in the YAML stream. + ## + ## **Warning:** The special float values ``[+-]Inf`` and ``NaN`` will be + ## parsed into Nim's JSON structure without error. However, they cannot be + ## rendered to a JSON character stream, because these values are not part + ## of the JSON specification. Nim's JSON implementation currently does not + ## check for these values and will output invalid JSON when rendering one + ## of these values into a JSON character stream. newSeq(result, 0) var @@ -153,7 +169,10 @@ proc constructJson*(s: var YamlStream): seq[JsonNode] = else: internalError("Unexpected node kind: " & $levels[levels.high].node.kind) -proc loadToJson*(s: Stream): seq[JsonNode] = +proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [YamlParserError].} = + ## Uses `YamlParser <#YamlParser>`_ and + ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree + ## from a YAML character stream. var parser = newYamlParser(initCoreTagLibrary()) events = parser.parse(s)