From 834c4b174ab452f49cd2b456234790e46b8b2576 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sat, 1 Oct 2016 15:19:42 +0200 Subject: [PATCH 01/26] doc link fixes --- yaml.nim | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/yaml.nim b/yaml.nim index 0703b65..a98843b 100644 --- a/yaml.nim +++ b/yaml.nim @@ -10,12 +10,12 @@ ## ## There are three high-level APIs which are probably most useful: ## -## * The serialization API in `serialization `_ enables +## * The serialization API in `serialization `_ enables ## you to load YAML data directly into native Nim types, and reversely dump ## native Nim types as YAML. -## * The DOM API in `dom `_ parses YAML files in a tree structure +## * The DOM API in `dom `_ parses YAML files in a tree structure ## which you can navigate. -## * The JSON API in `tojson `_ parses YAML files into the +## * The JSON API in `tojson `_ parses YAML files into the ## Nim stdlib's JSON structure, which may be useful if you have other modules ## which expect JSON input. Note that the serialization API is able to write ## and load JSON; you do not need the JSON API for that. @@ -24,18 +24,18 @@ ## enables you to process YAML input as data stream which does not need to be ## loaded into RAM completely at once. It consists of the following modules: ## -## * The stream API in `stream `_ defines the central type for +## * The stream API in `stream `_ defines the central type for ## stream processing, ``YamlStream``. It also contains definitions and ## constructor procs for stream events. -## * The parser API in `parser `_ gives you direct access to +## * The parser API in `parser `_ gives you direct access to ## the YAML parser's output. ## * The presenter API in `presenter `_ gives you direct ## access to the presenter, i.e. the module that renders a YAML character ## stream. -## * The taglib API in `taglib `_ provides a data structure +## * The taglib API in `taglib `_ provides a data structure ## for keeping track of YAML tags that are generated by the parser or used in ## the presenter. -## * The hints API in `hints `_ provides a simple proc for +## * The hints API in `hints `_ provides a simple proc for ## guessing the type of a scalar value. import yaml.dom, yaml.hints, yaml.parser, yaml.presenter, From 31ac201e413bc820a9c7658bd222b8a54f35a833 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sat, 8 Oct 2016 18:38:27 +0200 Subject: [PATCH 02/26] doc: extracted examples to own files --- .gitignore | 2 + config.nims | 14 +- doc/examples/customtag.nim | 14 + doc/examples/customtag.out.yaml | 5 + doc/examples/dump-json.nim | 13 + doc/examples/dump-json.out.yaml | 10 + doc/examples/dump-reftypes.nim | 20 ++ doc/examples/dump-reftypes.out.yaml | 11 + doc/examples/dump-yaml.nim | 12 + doc/examples/dump-yaml.out.yaml | 8 + doc/examples/implicit-variant.in.yaml | 8 + doc/examples/implicit-variant.nim | 37 ++ doc/examples/load-json.in.yaml | 10 + doc/examples/load-json.nim | 10 + doc/examples/load-reftypes.in.yaml | 11 + doc/examples/load-reftypes.nim | 12 + doc/examples/load-yaml.in.yaml | 4 + doc/examples/load-yaml.nim | 9 + doc/examples/outputstyle.nim | 16 + doc/examples/outputstyle.out.yaml | 16 + doc/examples/quickstart.json | 43 +++ doc/examples/sequential-api.in.yaml | 7 + doc/examples/sequential-api.nim | 51 +++ doc/index.txt | 463 +------------------------- doc/rstPreproc.nim | 127 +++++++ 25 files changed, 466 insertions(+), 467 deletions(-) create mode 100644 doc/examples/customtag.nim create mode 100644 doc/examples/customtag.out.yaml create mode 100644 doc/examples/dump-json.nim create mode 100644 doc/examples/dump-json.out.yaml create mode 100644 doc/examples/dump-reftypes.nim create mode 100644 doc/examples/dump-reftypes.out.yaml create mode 100644 doc/examples/dump-yaml.nim create mode 100644 doc/examples/dump-yaml.out.yaml create mode 100644 doc/examples/implicit-variant.in.yaml create mode 100644 doc/examples/implicit-variant.nim create mode 100644 doc/examples/load-json.in.yaml create mode 100644 doc/examples/load-json.nim create mode 100644 doc/examples/load-reftypes.in.yaml create mode 100644 doc/examples/load-reftypes.nim create mode 100644 doc/examples/load-yaml.in.yaml create mode 100644 doc/examples/load-yaml.nim create mode 100644 doc/examples/outputstyle.nim create mode 100644 doc/examples/outputstyle.out.yaml create mode 100644 doc/examples/quickstart.json create mode 100644 doc/examples/sequential-api.in.yaml create mode 100644 doc/examples/sequential-api.nim create mode 100644 doc/rstPreproc.nim diff --git a/.gitignore b/.gitignore index 2b7f03b..a345654 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,6 @@ libyaml.dylib libyaml.so bench/json docout +doc/rstPreproc +doc/tmp.rst yaml-dev-kit diff --git a/config.nims b/config.nims index 0d2610f..efa86f8 100644 --- a/config.nims +++ b/config.nims @@ -20,17 +20,21 @@ task serializationTests, "Run serialization tests": task documentation, "Generate documentation": exec "mkdir -p docout" + withDir "doc": + exec r"nim c rstPreproc" + exec r"./rstPreproc -o:tmp.rst index.txt" + exec r"nim rst2html -o:../docout/index.html tmp.rst" + exec r"./rstPreproc -o:tmp.rst api.txt" + exec r"nim rst2html -o:../docout/api.html tmp.rst" + exec r"./rstPreproc -o:tmp.rst serialization.txt" + exec r"nim rst2html -o:../docout/serialization.html tmp.rst" + exec "cp docutils.css style.css testing.html processing.svg ../docout" exec r"nim doc2 -o:docout/yaml.html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/`git log -n 1 --format=%H` yaml" - # bash! ah-ah \\ savior of the universe for file in listFiles("yaml"): let packageName = file[5..^5] exec r"nim doc2 -o:docout/yaml." & packageName & ".html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/yaml/`git log -n 1 --format=%H` " & file - exec r"nim rst2html -o:docout/index.html doc/index.txt" - exec r"nim rst2html -o:docout/api.html doc/api.txt" - exec r"nim rst2html -o:docout/serialization.html doc/serialization.txt" - exec "cp doc/docutils.css doc/style.css doc/testing.html doc/processing.svg docout" setCommand "nop" task bench, "Benchmarking": diff --git a/doc/examples/customtag.nim b/doc/examples/customtag.nim new file mode 100644 index 0000000..411fb58 --- /dev/null +++ b/doc/examples/customtag.nim @@ -0,0 +1,14 @@ +import yaml +type Mob = object + level, experience: int32 + drops: seq[string] + +setTagUri(Mob, "!Mob") +setTagUri(seq[string], "!Drops") + +var mob = Mob(level: 42, experience: 1800, drops: + @["Sword of Mob Slaying"]) +var s = newFileStream("out.yaml", fmWrite) +dump(mob, s, + options = defineOptions(tagStyle = tsAll)) +s.close() \ No newline at end of file diff --git a/doc/examples/customtag.out.yaml b/doc/examples/customtag.out.yaml new file mode 100644 index 0000000..cf6deba --- /dev/null +++ b/doc/examples/customtag.out.yaml @@ -0,0 +1,5 @@ +%YAML 1.2 +--- !Mob +!nim:field level: !nim:system:int32 42 +!nim:field experience: !nim:system:int32 1800 +!nim:field drops: !Drops [!!str Sword of Mob Slaying] \ No newline at end of file diff --git a/doc/examples/dump-json.nim b/doc/examples/dump-json.nim new file mode 100644 index 0000000..7b91553 --- /dev/null +++ b/doc/examples/dump-json.nim @@ -0,0 +1,13 @@ +import yaml +type Person = object + name : string + age : int32 + +var personList = newSeq[Person]() +personList.add(Person(name: "Karl Koch", age: 23)) +personList.add(Person(name: "Peter Pan", age: 12)) + +var s = newFileStream("out.yaml", fmWrite) +dump(personList, s, + options = defineOptions(style = psJson)) +s.close() \ No newline at end of file diff --git a/doc/examples/dump-json.out.yaml b/doc/examples/dump-json.out.yaml new file mode 100644 index 0000000..3be727c --- /dev/null +++ b/doc/examples/dump-json.out.yaml @@ -0,0 +1,10 @@ +[ + { + "name": "Karl Koch", + "age": 23 + }, + { + "name": "Peter Pan", + "age": 12 + } +] \ No newline at end of file diff --git a/doc/examples/dump-reftypes.nim b/doc/examples/dump-reftypes.nim new file mode 100644 index 0000000..0e97b6c --- /dev/null +++ b/doc/examples/dump-reftypes.nim @@ -0,0 +1,20 @@ +import yaml +type + Node = ref NodeObj + NodeObj = object + name: string + left, right: Node + +var node1, node2, node3: Node +new(node1); new(node2); new(node3) +node1.name = "Node 1" +node2.name = "Node 2" +node3.name = "Node 3" +node1.left = node2 +node1.right = node3 +node2.right = node3 +node3.left = node1 + +var s = newFileStream("out.yaml", fmWrite) +dump(node1, s) +s.close() \ No newline at end of file diff --git a/doc/examples/dump-reftypes.out.yaml b/doc/examples/dump-reftypes.out.yaml new file mode 100644 index 0000000..25e569e --- /dev/null +++ b/doc/examples/dump-reftypes.out.yaml @@ -0,0 +1,11 @@ +%YAML 1.2 +--- !nim:custom:NodeObj &a +name: Node 1 +left: + name: Node 2 + left: !!null ~ + right: &b + name: Node 3 + left: *a + right: !!null ~ +right: *b \ No newline at end of file diff --git a/doc/examples/dump-yaml.nim b/doc/examples/dump-yaml.nim new file mode 100644 index 0000000..b1bec26 --- /dev/null +++ b/doc/examples/dump-yaml.nim @@ -0,0 +1,12 @@ +import yaml +type Person = object + name : string + age : int32 + +var personList = newSeq[Person]() +personList.add(Person(name: "Karl Koch", age: 23)) +personList.add(Person(name: "Peter Pan", age: 12)) + +var s = newFileStream("out.yaml", fmWrite) +dump(personList, s) +s.close() \ No newline at end of file diff --git a/doc/examples/dump-yaml.out.yaml b/doc/examples/dump-yaml.out.yaml new file mode 100644 index 0000000..46d0583 --- /dev/null +++ b/doc/examples/dump-yaml.out.yaml @@ -0,0 +1,8 @@ +%YAML 1.2 +--- !nim:system:seq(nim:custom:Person) +- + name: Karl Koch + age: 23 +- + name: Peter Pan + age: 12 \ No newline at end of file diff --git a/doc/examples/implicit-variant.in.yaml b/doc/examples/implicit-variant.in.yaml new file mode 100644 index 0000000..aae04fc --- /dev/null +++ b/doc/examples/implicit-variant.in.yaml @@ -0,0 +1,8 @@ +%YAML 1.2 +--- +- this is a string +- 42 +- false +- !!str 23 +- !nim:demo:Person {name: Trillian} +- !!null \ No newline at end of file diff --git a/doc/examples/implicit-variant.nim b/doc/examples/implicit-variant.nim new file mode 100644 index 0000000..323c410 --- /dev/null +++ b/doc/examples/implicit-variant.nim @@ -0,0 +1,37 @@ +import yaml +type + Person = object + name: string + + ContainerKind = enum + ckString, ckInt, ckBool, ckPerson, ckNone + + Container = object + case kind: ContainerKind + of ckString: + strVal: string + of ckInt: + intVal: int + of ckBool: + boolVal: bool + of ckPerson: + personVal: Person + of ckNone: + discard + +setTagUri(Person, "!nim:demo:Person") + +# tell NimYAML to use Container as implicit type. +# only possible with variant object types where +# each branch contains at most one object. +markAsImplicit(Container) + +var list: seq[Container] + +var s = newFileStream("in.yaml") +load(s, list) +s.close() + +assert(list[0].kind == ckString) +assert(list[0].strVal == "this is a string") +# and so on \ No newline at end of file diff --git a/doc/examples/load-json.in.yaml b/doc/examples/load-json.in.yaml new file mode 100644 index 0000000..3be727c --- /dev/null +++ b/doc/examples/load-json.in.yaml @@ -0,0 +1,10 @@ +[ + { + "name": "Karl Koch", + "age": 23 + }, + { + "name": "Peter Pan", + "age": 12 + } +] \ No newline at end of file diff --git a/doc/examples/load-json.nim b/doc/examples/load-json.nim new file mode 100644 index 0000000..cf86f03 --- /dev/null +++ b/doc/examples/load-json.nim @@ -0,0 +1,10 @@ +import yaml +type Person = object + name : string + age : int32 + +var personList: seq[Person] + +var s = newFileStream("in.yaml") +load(s, personList) +s.close() \ No newline at end of file diff --git a/doc/examples/load-reftypes.in.yaml b/doc/examples/load-reftypes.in.yaml new file mode 100644 index 0000000..78d163b --- /dev/null +++ b/doc/examples/load-reftypes.in.yaml @@ -0,0 +1,11 @@ +%YAML 1.2 +--- !nim:custom:NodeObj &a +name: Node 1 +left: + name: Node 2 + left: ~ + right: &b + name: Node 3 + left: *a + right: ~ +right: *b \ No newline at end of file diff --git a/doc/examples/load-reftypes.nim b/doc/examples/load-reftypes.nim new file mode 100644 index 0000000..983ae94 --- /dev/null +++ b/doc/examples/load-reftypes.nim @@ -0,0 +1,12 @@ +import yaml +type + Node = ref NodeObj + NodeObj = object + name: string + left, right: Node + +var node1: Node + +var s = newFileStream("in.yaml") +load(s, node1) +s.close() \ No newline at end of file diff --git a/doc/examples/load-yaml.in.yaml b/doc/examples/load-yaml.in.yaml new file mode 100644 index 0000000..b918cbf --- /dev/null +++ b/doc/examples/load-yaml.in.yaml @@ -0,0 +1,4 @@ +%YAML 1.2 +--- +- { name: Karl Koch, age: 23 } +- { name: Peter Pan, age: 12 } \ No newline at end of file diff --git a/doc/examples/load-yaml.nim b/doc/examples/load-yaml.nim new file mode 100644 index 0000000..3ceb4ce --- /dev/null +++ b/doc/examples/load-yaml.nim @@ -0,0 +1,9 @@ +import yaml +type Person = object + name : string + age : int32 + +var personList: seq[Person] +var s = newFileStream("in.yaml") +load(s, personList) +s.close() \ No newline at end of file diff --git a/doc/examples/outputstyle.nim b/doc/examples/outputstyle.nim new file mode 100644 index 0000000..2e80785 --- /dev/null +++ b/doc/examples/outputstyle.nim @@ -0,0 +1,16 @@ +import yaml +type Person = object + name: string + age: int32 + +var personList: seq[Person] +personList.add(Person(name: "Karl Koch", age: 23)) +personList.add(Person(name: "Peter Pan", age: 12)) + +var s = newFileStream("out.yaml") +dump(personList, s, options = defineOptions( + style = psCanonical, + indentationStep = 3, + newlines = nlLF, + outputVersion = ov1_1)) +s.close() \ No newline at end of file diff --git a/doc/examples/outputstyle.out.yaml b/doc/examples/outputstyle.out.yaml new file mode 100644 index 0000000..76b4fc0 --- /dev/null +++ b/doc/examples/outputstyle.out.yaml @@ -0,0 +1,16 @@ +%YAML 1.1 +--- !nim:system:seq(nim:custom:Person) +[ + !nim:custom:Person { + ? !!str "name" + : !!str "Karl Koch", + ? !!str "age" + : !nim:system:int32 "23" + }, + !nim:custom:Person { + ? !!str "name" + : !!str "Peter Pan", + ? !!str "age" + : !nim:system:int32 "12" + } +] \ No newline at end of file diff --git a/doc/examples/quickstart.json b/doc/examples/quickstart.json new file mode 100644 index 0000000..b8efe1c --- /dev/null +++ b/doc/examples/quickstart.json @@ -0,0 +1,43 @@ +{ + "Dumping Nim objects as YAML": [ + "dump-yaml", "out" + ], + + "Loading Nim objects from YAML": [ + "load-yaml", "in" + ], + + "Customizing output style" : [ + "outputstyle", "out" + ], + + "Dumping reference types and cyclic structures": [ + "dump-reftypes", "out" + ], + + "Loading reference types and cyclic structures": [ + "load-reftypes", "in" + ], + + "Defining a custom tag uri for a type": [ + "customtag", "out" + ], + + "Dumping Nim objects as JSON": [ + "dump-json", "out" + ], + + "Loading Nim objects from JSON": [ + "load-json", "in" + ], + + "Processing a Sequence of Heterogeneous Items": { + "… With variant objects": [ + "implicit-variant", "in" + ], + + "… With the Sequential API": [ + "sequential-api", "in" + ] + } +} \ No newline at end of file diff --git a/doc/examples/sequential-api.in.yaml b/doc/examples/sequential-api.in.yaml new file mode 100644 index 0000000..8164e19 --- /dev/null +++ b/doc/examples/sequential-api.in.yaml @@ -0,0 +1,7 @@ +%YAML 1.2 +--- !!seq +- this is a string +- 42 +- false +- !!str 23 +- !nim:demo:Person {name: Trillian} \ No newline at end of file diff --git a/doc/examples/sequential-api.nim b/doc/examples/sequential-api.nim new file mode 100644 index 0000000..e1ada99 --- /dev/null +++ b/doc/examples/sequential-api.nim @@ -0,0 +1,51 @@ +import yaml +type Person = object + name: string + +setTagUri(Person, "!nim:demo:Person", yTagPerson) + +var + s = newFileStream("in.yaml", fmRead) + context = newConstructionContext() + parser = newYamlParser(serializationTagLibrary) + events = parser.parse(s) + +assert events.next().kind == yamlStartDoc +assert events.next().kind == yamlStartSeq +var nextEvent = events.peek() +while nextEvent.kind != yamlEndSeq: + var curTag = nextEvent.tag() + if curTag == yTagQuestionMark: + # we only support implicitly tagged scalars + assert nextEvent.kind == yamlScalar + case guessType(nextEvent.scalarContent) + of yTypeInteger: curTag = yTagInteger + of yTypeBoolTrue, yTypeBoolFalse: + curTag = yTagBoolean + of yTypeUnknown: curTag = yTagString + else: assert false, "Type not supported!" + elif curTag == yTagExclamationMark: + curTag = yTagString + case curTag + of yTagString: + var s: string + events.constructChild(context, s) + echo "got string: ", s + of yTagInteger: + var i: int32 + events.constructChild(context, i) + echo "got integer: ", i + of yTagBoolean: + var b: bool + events.constructChild(context, b) + echo "got boolean: ", b + of yTagPerson: + var p: Person + events.constructChild(context, p) + echo "got Person with name: ", p.name + else: assert false, "unsupported tag: " & $curTag + nextEvent = events.peek() +assert events.next().kind == yamlEndSeq +assert events.next().kind == yamlEndDoc +assert events.finished() +s.close() \ No newline at end of file diff --git a/doc/index.txt b/doc/index.txt index 57f6080..3bd71cd 100644 --- a/doc/index.txt +++ b/doc/index.txt @@ -19,465 +19,4 @@ install it with `Nimble `_: Quickstart ========== -Dumping Nim objects as YAML --------------------------- - -.. raw:: html - - -
code.nimout.yaml
- -.. code-block:: nim - import yaml - type Person = object - name : string - age : int32 - - var personList = newSeq[Person]() - personList.add(Person(name: "Karl Koch", age: 23)) - personList.add(Person(name: "Peter Pan", age: 12)) - - var s = newFileStream("out.yaml", fmWrite) - dump(personList, s) - s.close() - -.. raw:: html - - -.. code-block:: yaml - %YAML 1.2 - --- !nim:system:seq(nim:custom:Person) - - - name: Karl Koch - age: 23 - - - name: Peter Pan - age: 12 - -.. raw:: html -
- -Loading Nim objects from YAML ----------------------------- - -.. raw:: html - - -
code.nimin.yaml
- -.. code-block:: nim - import yaml - type Person = object - name : string - age : int32 - - var personList: seq[Person] - var s = newFileStream("in.yaml") - load(s, personList) - s.close() - -.. raw:: html - - -.. code-block:: yaml - %YAML 1.2 - --- - - { name: Karl Koch, age: 23 } - - { name: Peter Pan, age: 12 } - -.. raw:: html -
- -Customizing output style ----------------------- - -.. raw:: html - -
code.nimout.yaml
- -.. code-block:: nim - import yaml - type Person = object - name: string - age: int32 - - var personList: seq[Person] - personList.add(Person(name: "Karl Koch", age: 23)) - personList.add(Person(name: "Peter Pan", age: 12)) - - var s = newFileStream("out.yaml") - dump(personList, s, options = defineOptions( - style = psCanonical, - indentationStep = 3, - newlines = nlLF, - outputVersion = ov1_1)) - s.close() - -.. raw:: html - - -.. code-block:: yaml - %YAML 1.1 - --- !nim:system:seq(nim:custom:Person) - [ - !nim:custom:Person { - ? !!str "name" - : !!str "Karl Koch", - ? !!str "age" - : !nim:system:int32 "23" - }, - !nim:custom:Person { - ? !!str "name" - : !!str "Peter Pan", - ? !!str "age" - : !nim:system:int32 "12" - } - ] - -.. raw:: html -
- -Dumping reference types and cyclic structures ---------------------------------------------- - -.. raw:: html - -
code.nimout.yaml
- -.. code-block:: nim - import yaml - type - Node = ref NodeObj - NodeObj = object - name: string - left, right: Node - - var node1, node2, node3: Node - new(node1); new(node2); new(node3) - node1.name = "Node 1" - node2.name = "Node 2" - node3.name = "Node 3" - node1.left = node2 - node1.right = node3 - node2.right = node3 - node3.left = node1 - - var s = newFileStream("out.yaml", fmWrite) - dump(node1, s) - s.close() - -.. raw:: html - - -.. code-block:: yaml - %YAML 1.2 - --- !nim:custom:NodeObj &a - name: Node 1 - left: - name: Node 2 - left: !!null ~ - right: &b - name: Node 3 - left: *a - right: !!null ~ - right: *b - -.. raw:: html -
- -Loading reference types and cyclic structures ---------------------------------------------- - -.. raw:: html - -
code.nimin.yaml
- -.. code-block:: nim - import yaml - type - Node = ref NodeObj - NodeObj = object - name: string - left, right: Node - - var node1: Node - - var s = newFileStream("in.yaml") - load(s, node1) - s.close() - -.. raw:: html - - -.. code-block:: yaml - %YAML 1.2 - --- !nim:custom:NodeObj &a - name: Node 1 - left: - name: Node 2 - left: ~ - right: &b - name: Node 3 - left: *a - right: ~ - right: *b - -.. raw:: html -
- -Defining a custom tag uri for a type ------------------------------------- - -.. raw:: html - -
code.nimout.yaml
- -.. code-block:: nim - import yaml - type Mob = object - level, experience: int32 - drops: seq[string] - - setTagUri(Mob, "!Mob") - setTagUri(seq[string], "!Drops") - - var mob = Mob(level: 42, experience: 1800, drops: - @["Sword of Mob Slaying"]) - var s = newFileStream("out.yaml", fmWrite) - dump(mob, s, - options = defineOptions(tagStyle = tsAll)) - s.close() - -.. raw:: html - - -.. code-block:: yaml - YAML 1.2 - --- !Mob - !nim:field level: !nim:system:int32 42 - !nim:field experience: !nim:system:int32 1800 - !nim:field drops: !Drops [!!str Sword of Mob Slaying] - -.. raw:: html -
- -Dumping Nim objects as JSON ---------------------------- - -.. raw:: html - - -
code.nimout.yaml
- -.. code-block:: nim - import yaml - type Person = object - name : string - age : int32 - - var personList = newSeq[Person]() - personList.add(Person(name: "Karl Koch", age: 23)) - personList.add(Person(name: "Peter Pan", age: 12)) - - var s = newFileStream("out.yaml", fmWrite) - dump(personList, s, - options = defineOptions(style = psJson)) - s.close() - -.. raw:: html - - -.. code-block:: yaml - [ - { - "name": "Karl Koch", - "age": 23 - }, - { - "name": "Peter Pan", - "age": 12 - } - ] - -.. raw:: html -
- -Loading Nim objects from JSON ------------------------------ - -.. raw:: html - - -
code.nimin.yaml
- -.. code-block:: nim - import yaml - type Person = object - name : string - age : int32 - - var personList: seq[Person] - - var s = newFileStream("in.yaml") - load(s, personList) - s.close() - -.. raw:: html - - -.. code-block:: yaml - [ - { - "name": "Karl Koch", - "age": 23 - }, - { - "name": "Peter Pan", - "age": 12 - } - ] - -.. raw:: html -
- -Processing a Sequence of Heterogeneous Items --------------------------------------------- - -… With variant objects -...................... - -.. raw:: html - - -
code.nimin.yaml
- -.. code-block:: nim - import yaml - type - Person = object - name: string - - ContainerKind = enum - ckString, ckInt, ckBool, ckPerson, ckNone - - Container = object - case kind: ContainerKind - of ckString: - strVal: string - of ckInt: - intVal: int - of ckBool: - boolVal: bool - of ckPerson: - personVal: Person - of ckNone: - discard - - setTagUri(Person, "!nim:demo:Person") - - # tell NimYAML to use Container as implicit type. - # only possible with variant object types where - # each branch contains at most one object. - markAsImplicit(Container) - - var list: seq[Container] - - var s = newFileStream("in.yaml") - load(s, list) - s.close() - - assert(list[0].kind == ckString) - assert(list[0].strVal == "this is a string") - # and so on - -.. raw:: html - - -.. code-block:: yaml - %YAML 1.2 - --- - - this is a string - - 42 - - false - - !!str 23 - - !nim:demo:Person {name: Trillian} - - !!null - -.. raw:: html -
- -… With the Sequential API -......................... - -.. raw:: html - - -
code.nimin.yaml
- -.. code-block:: nim - import yaml - type Person = object - name: string - - setTagUri(Person, "!nim:demo:Person", yTagPerson) - - var - s = newFileStream("in.yaml", fmRead) - context = newConstructionContext() - parser = newYamlParser(serializationTagLibrary) - events = parser.parse(s) - - assert events.next().kind == yamlStartDoc - assert events.next().kind == yamlStartSeq - var nextEvent = events.peek() - while nextEvent.kind != yamlEndSeq: - var curTag = nextEvent.tag() - if curTag == yTagQuestionMark: - # we only support implicitly tagged scalars - assert nextEvent.kind == yamlScalar - case guessType(nextEvent.scalarContent) - of yTypeInteger: curTag = yTagInteger - of yTypeBoolTrue, yTypeBoolFalse: - curTag = yTagBoolean - of yTypeUnknown: curTag = yTagString - else: assert false, "Type not supported!" - elif curTag == yTagExclamationMark: - curTag = yTagString - case curTag - of yTagString: - var s: string - events.constructChild(context, s) - echo "got string: ", s - of yTagInteger: - var i: int32 - events.constructChild(context, i) - echo "got integer: ", i - of yTagBoolean: - var b: bool - events.constructChild(context, b) - echo "got boolean: ", b - of yTagPerson: - var p: Person - events.constructChild(context, p) - echo "got Person with name: ", p.name - else: assert false, "unsupported tag: " & $curTag - nextEvent = events.peek() - assert events.next().kind == yamlEndSeq - assert events.next().kind == yamlEndDoc - assert events.finished() - s.close() - -.. raw:: html - - -.. code-block:: yaml - %YAML 1.2 - --- !!seq - - this is a string - - 42 - - false - - !!str 23 - - !nim:demo:Person {name: Trillian} - -.. raw:: html -
\ No newline at end of file +%examples/quickstart.json%/ \ No newline at end of file diff --git a/doc/rstPreproc.nim b/doc/rstPreproc.nim new file mode 100644 index 0000000..6b92460 --- /dev/null +++ b/doc/rstPreproc.nim @@ -0,0 +1,127 @@ +## This is a tool for preprocessing rst files. Lines starting with ``%`` will +## get substituted by nicely layouted included nim and yaml code. +## +## The syntax of substituted lines is ``'%' jsonfile '%' jsonpath``. *jsonfile* +## shall be the path to a JSON file. *jsonpath* shall be a path to some node in +## that JSON file. +## +## Usage: +## +## rstPreproc -o:path +## +## *path* is the output path. If omitted, it will be equal to infile with its +## suffix substituted by ``.rst``. *infile* is the source rst file. + +import parseopt2, json, streams, tables, strutils, os + +var + infile = "" + path: string = nil +for kind, key, val in getopt(): + case kind + of cmdArgument: + if infile == "": + if key == "": + echo "invalid input file with empty name!" + quit 1 + infile = key + else: + echo "Only one input file is supported!" + quit 1 + of cmdLongOption, cmdShortOption: + case key + of "out", "o": + if isNil(path): path = val + else: + echo "Duplicate output path!" + quit 1 + else: + echo "Unknown option: ", key + quit 1 + of cmdEnd: assert(false) # cannot happen + +if infile == "": + echo "Missing input file!" + quit 1 + +if isNil(path): + for i in countdown(infile.len - 1, 0): + if infile[i] == '.': + if infile[i..^1] == ".rst": path = infile & ".rst" + else: path = infile[0..i] & "rst" + break + if isNil(path): path = infile & ".rst" + +var tmpOut = newFileStream(path, fmWrite) + +proc append(s: string) = + tmpOut.writeLine(s) + +proc gotoPath(root: JsonNode, path: string): JsonNode = + doAssert path[0] == '/' + if path.len == 1: return root + doAssert root.kind == JObject + for i in 1..= headingChars.len: headingChars[^1] else: + headingChars[level] + append(repeat(headingChar, key.len) & '\l') + outputExamples(value, prefix, level + 1) + of JArray: + let elems = node.getElems() + case elems.len + of 2: + append(".. raw:: html") + append(" ") + append(" \n
code.nim" & elems[1].getStr() & + ".yaml
\n") + append(".. code:: nim") + append(" :file: " & prefix & elems[0].getStr() & ".nim\n") + append(".. raw:: html") + append(" \n") + append(".. code:: yaml") + append(" :file: " & prefix & elems[0].getStr() & '.' & + elems[1].getStr() & ".yaml\n") + append(".. raw:: html") + append("
\n") + else: + echo "Unexpected number of elements in array: ", elems.len + quit 1 + else: + echo "Unexpected node kind: ", node.kind + quit 1 + +var lineNum = 0 +for line in infile.lines(): + if line.len > 0 and line[0] == '%': + var + jsonFile: string = nil + jsonPath: string = nil + for i in 1.. Date: Sat, 8 Oct 2016 20:57:53 +0200 Subject: [PATCH 03/26] Ordered snippets with file system hierarchy --- doc/examples/quickstart.json | 43 ------- doc/index.txt | 5 +- doc/rstPreproc.nim | 119 +++++++++--------- .../quickstart/01/code.nim} | 0 .../quickstart/01/out.yaml} | 0 doc/snippets/quickstart/01/title | 1 + .../quickstart/02/code.nim} | 0 .../quickstart/02/in.yaml} | 0 doc/snippets/quickstart/02/title | 1 + .../quickstart/03/code.nim} | 0 .../quickstart/03/out.yaml} | 0 doc/snippets/quickstart/03/title | 1 + .../quickstart/04/code.nim} | 0 .../quickstart/04/out.yaml} | 0 doc/snippets/quickstart/04/title | 1 + .../quickstart/05/code.nim} | 0 .../quickstart/05/in.yaml} | 0 doc/snippets/quickstart/05/title | 1 + .../quickstart/06/code.nim} | 0 .../quickstart/06/out.yaml} | 0 doc/snippets/quickstart/06/title | 1 + .../quickstart/07/code.nim} | 0 .../quickstart/07/out.yaml} | 0 doc/snippets/quickstart/07/title | 1 + .../quickstart/08/code.nim} | 0 .../quickstart/08/in.yaml} | 0 doc/snippets/quickstart/08/title | 1 + .../quickstart/09/01/code.nim} | 0 .../quickstart/09/01/in.yaml} | 0 doc/snippets/quickstart/09/01/title | 1 + .../quickstart/09/02/code.nim} | 0 .../quickstart/09/02/in.yaml} | 0 doc/snippets/quickstart/09/02/title | 1 + doc/snippets/quickstart/09/title | 1 + doc/snippets/quickstart/title | 1 + yaml/serialization.nim | 44 ++++++- yaml/taglib.nim | 26 ---- 37 files changed, 116 insertions(+), 133 deletions(-) delete mode 100644 doc/examples/quickstart.json rename doc/{examples/dump-yaml.nim => snippets/quickstart/01/code.nim} (100%) rename doc/{examples/dump-yaml.out.yaml => snippets/quickstart/01/out.yaml} (100%) create mode 100644 doc/snippets/quickstart/01/title rename doc/{examples/load-yaml.nim => snippets/quickstart/02/code.nim} (100%) rename doc/{examples/load-yaml.in.yaml => snippets/quickstart/02/in.yaml} (100%) create mode 100644 doc/snippets/quickstart/02/title rename doc/{examples/outputstyle.nim => snippets/quickstart/03/code.nim} (100%) rename doc/{examples/outputstyle.out.yaml => snippets/quickstart/03/out.yaml} (100%) create mode 100644 doc/snippets/quickstart/03/title rename doc/{examples/dump-reftypes.nim => snippets/quickstart/04/code.nim} (100%) rename doc/{examples/dump-reftypes.out.yaml => snippets/quickstart/04/out.yaml} (100%) create mode 100644 doc/snippets/quickstart/04/title rename doc/{examples/load-reftypes.nim => snippets/quickstart/05/code.nim} (100%) rename doc/{examples/load-reftypes.in.yaml => snippets/quickstart/05/in.yaml} (100%) create mode 100644 doc/snippets/quickstart/05/title rename doc/{examples/customtag.nim => snippets/quickstart/06/code.nim} (100%) rename doc/{examples/customtag.out.yaml => snippets/quickstart/06/out.yaml} (100%) create mode 100644 doc/snippets/quickstart/06/title rename doc/{examples/dump-json.nim => snippets/quickstart/07/code.nim} (100%) rename doc/{examples/dump-json.out.yaml => snippets/quickstart/07/out.yaml} (100%) create mode 100644 doc/snippets/quickstart/07/title rename doc/{examples/load-json.nim => snippets/quickstart/08/code.nim} (100%) rename doc/{examples/load-json.in.yaml => snippets/quickstart/08/in.yaml} (100%) create mode 100644 doc/snippets/quickstart/08/title rename doc/{examples/implicit-variant.nim => snippets/quickstart/09/01/code.nim} (100%) rename doc/{examples/implicit-variant.in.yaml => snippets/quickstart/09/01/in.yaml} (100%) create mode 100644 doc/snippets/quickstart/09/01/title rename doc/{examples/sequential-api.nim => snippets/quickstart/09/02/code.nim} (100%) rename doc/{examples/sequential-api.in.yaml => snippets/quickstart/09/02/in.yaml} (100%) create mode 100644 doc/snippets/quickstart/09/02/title create mode 100644 doc/snippets/quickstart/09/title create mode 100644 doc/snippets/quickstart/title diff --git a/doc/examples/quickstart.json b/doc/examples/quickstart.json deleted file mode 100644 index b8efe1c..0000000 --- a/doc/examples/quickstart.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "Dumping Nim objects as YAML": [ - "dump-yaml", "out" - ], - - "Loading Nim objects from YAML": [ - "load-yaml", "in" - ], - - "Customizing output style" : [ - "outputstyle", "out" - ], - - "Dumping reference types and cyclic structures": [ - "dump-reftypes", "out" - ], - - "Loading reference types and cyclic structures": [ - "load-reftypes", "in" - ], - - "Defining a custom tag uri for a type": [ - "customtag", "out" - ], - - "Dumping Nim objects as JSON": [ - "dump-json", "out" - ], - - "Loading Nim objects from JSON": [ - "load-json", "in" - ], - - "Processing a Sequence of Heterogeneous Items": { - "… With variant objects": [ - "implicit-variant", "in" - ], - - "… With the Sequential API": [ - "sequential-api", "in" - ] - } -} \ No newline at end of file diff --git a/doc/index.txt b/doc/index.txt index 3bd71cd..d1a7498 100644 --- a/doc/index.txt +++ b/doc/index.txt @@ -16,7 +16,4 @@ install it with `Nimble `_: .. code-block:: bash nimble install yaml -Quickstart -========== - -%examples/quickstart.json%/ \ No newline at end of file +%quickstart%0 \ No newline at end of file diff --git a/doc/rstPreproc.nim b/doc/rstPreproc.nim index 6b92460..22522eb 100644 --- a/doc/rstPreproc.nim +++ b/doc/rstPreproc.nim @@ -1,9 +1,10 @@ ## This is a tool for preprocessing rst files. Lines starting with ``%`` will -## get substituted by nicely layouted included nim and yaml code. +## get substituted by nicely layouted nim and yaml code included from file in +## the snippets tree. ## -## The syntax of substituted lines is ``'%' jsonfile '%' jsonpath``. *jsonfile* -## shall be the path to a JSON file. *jsonpath* shall be a path to some node in -## that JSON file. +## The syntax of substituted lines is ``'%' path '%' level``. *path* shall be +## a path relative to the *snippets* directory. *level* shall be the level depth +## of the first title that should be produced. ## ## Usage: ## @@ -11,8 +12,12 @@ ## ## *path* is the output path. If omitted, it will be equal to infile with its ## suffix substituted by ``.rst``. *infile* is the source rst file. +## +## The reason for this complex approach is to have all snippets used in the docs +## available as source files for automatic testing. This way, we can make sure +## that the code in the docs actually works. -import parseopt2, json, streams, tables, strutils, os +import parseopt2, streams, tables, strutils, os var infile = "" @@ -57,71 +62,73 @@ var tmpOut = newFileStream(path, fmWrite) proc append(s: string) = tmpOut.writeLine(s) -proc gotoPath(root: JsonNode, path: string): JsonNode = - doAssert path[0] == '/' - if path.len == 1: return root - doAssert root.kind == JObject - for i in 1..= headingChars.len: headingChars[^1] else: headingChars[level] - append(repeat(headingChar, key.len) & '\l') - outputExamples(value, prefix, level + 1) - of JArray: - let elems = node.getElems() - case elems.len - of 2: - append(".. raw:: html") - append(" ") - append(" \n
code.nim" & elems[1].getStr() & - ".yaml
\n") - append(".. code:: nim") - append(" :file: " & prefix & elems[0].getStr() & ".nim\n") - append(".. raw:: html") - append(" \n") - append(".. code:: yaml") - append(" :file: " & prefix & elems[0].getStr() & '.' & - elems[1].getStr() & ".yaml\n") - append(".. raw:: html") - append("
\n") - else: - echo "Unexpected number of elements in array: ", elems.len - quit 1 + append(title) + append(repeat(headingChar, title.len) & '\l') + + # process content files under this directory + + var codeFiles = newSeq[string]() + for kind, filePath in walkDir(curPath, true): + if kind == pcFile: + if filePath != "title": codeFiles.add(filePath) + case codeFiles.len + of 0: discard + of 1: + let (nullPath, name, extension) = codeFiles[0].splitFile() + append(".. code:: " & extension[1..^1]) + append(" :file: " & (curPath / codeFiles[0]) & '\l') + of 2: + append(".. raw:: html") + append(" ") + for codeFile in codeFiles: + append(" ") + append(" \n
" & codeFile & "
\n") + + var first = true + for codeFile in codeFiles: + if first: first = false + else: append(".. raw:: html\n \n") + let (nullPath, name, extension) = codeFile.splitFile() + append(".. code:: " & extension[1..^1]) + append(" :file: " & (curPath / codeFile) & '\l') + + append(".. raw:: html") + append("
\n") else: - echo "Unexpected node kind: ", node.kind - quit 1 + echo "Unexpected number of files in ", curPath, ": ", codeFiles.len + + # process child directories + + for kind, dirPath in walkDir(curPath): + if kind == pcDir: + outputExamples(dirPath, level + 1) var lineNum = 0 for line in infile.lines(): if line.len > 0 and line[0] == '%': var - jsonFile: string = nil - jsonPath: string = nil + srcPath: string = nil + level = 0 for i in 1.. Date: Sat, 8 Oct 2016 21:08:12 +0200 Subject: [PATCH 04/26] Numeric sorting also for snippet files --- doc/rstPreproc.nim | 6 +++--- doc/snippets/quickstart/{01/code.nim => 00/00-code.nim} | 0 doc/snippets/quickstart/{01/out.yaml => 00/01-out.yaml} | 0 doc/snippets/quickstart/00/title | 1 + doc/snippets/quickstart/{02/code.nim => 01/00-code.nim} | 0 doc/snippets/quickstart/{02/in.yaml => 01/01-in.yaml} | 0 doc/snippets/quickstart/01/title | 2 +- doc/snippets/quickstart/{03/code.nim => 02/00-code.nim} | 0 doc/snippets/quickstart/{03/out.yaml => 02/01-out.yaml} | 0 doc/snippets/quickstart/02/title | 2 +- doc/snippets/quickstart/{04/code.nim => 03/00-code.nim} | 0 doc/snippets/quickstart/{04/out.yaml => 03/01-out.yaml} | 0 doc/snippets/quickstart/03/title | 2 +- doc/snippets/quickstart/{05/code.nim => 04/00-code.nim} | 0 doc/snippets/quickstart/{05/in.yaml => 04/01-in.yaml} | 0 doc/snippets/quickstart/04/title | 2 +- doc/snippets/quickstart/{06/code.nim => 05/00-code.nim} | 0 doc/snippets/quickstart/{06/out.yaml => 05/01-out.yaml} | 0 doc/snippets/quickstart/05/title | 2 +- doc/snippets/quickstart/{07/code.nim => 06/00-code.nim} | 0 doc/snippets/quickstart/{07/out.yaml => 06/01-out.yaml} | 0 doc/snippets/quickstart/06/title | 2 +- doc/snippets/quickstart/{08/code.nim => 07/00-code.nim} | 0 doc/snippets/quickstart/{08/in.yaml => 07/01-in.yaml} | 0 doc/snippets/quickstart/07/title | 2 +- .../quickstart/{09/01/code.nim => 08/00/00-code.nim} | 0 doc/snippets/quickstart/{09/01/in.yaml => 08/00/01-in.yaml} | 0 doc/snippets/quickstart/{09/01 => 08/00}/title | 0 .../quickstart/{09/02/code.nim => 08/01/00-code.nim} | 0 doc/snippets/quickstart/{09/02/in.yaml => 08/01/01-in.yaml} | 0 doc/snippets/quickstart/{09/02 => 08/01}/title | 0 doc/snippets/quickstart/08/title | 2 +- doc/snippets/quickstart/09/title | 1 - 33 files changed, 12 insertions(+), 12 deletions(-) rename doc/snippets/quickstart/{01/code.nim => 00/00-code.nim} (100%) rename doc/snippets/quickstart/{01/out.yaml => 00/01-out.yaml} (100%) create mode 100644 doc/snippets/quickstart/00/title rename doc/snippets/quickstart/{02/code.nim => 01/00-code.nim} (100%) rename doc/snippets/quickstart/{02/in.yaml => 01/01-in.yaml} (100%) rename doc/snippets/quickstart/{03/code.nim => 02/00-code.nim} (100%) rename doc/snippets/quickstart/{03/out.yaml => 02/01-out.yaml} (100%) rename doc/snippets/quickstart/{04/code.nim => 03/00-code.nim} (100%) rename doc/snippets/quickstart/{04/out.yaml => 03/01-out.yaml} (100%) rename doc/snippets/quickstart/{05/code.nim => 04/00-code.nim} (100%) rename doc/snippets/quickstart/{05/in.yaml => 04/01-in.yaml} (100%) rename doc/snippets/quickstart/{06/code.nim => 05/00-code.nim} (100%) rename doc/snippets/quickstart/{06/out.yaml => 05/01-out.yaml} (100%) rename doc/snippets/quickstart/{07/code.nim => 06/00-code.nim} (100%) rename doc/snippets/quickstart/{07/out.yaml => 06/01-out.yaml} (100%) rename doc/snippets/quickstart/{08/code.nim => 07/00-code.nim} (100%) rename doc/snippets/quickstart/{08/in.yaml => 07/01-in.yaml} (100%) rename doc/snippets/quickstart/{09/01/code.nim => 08/00/00-code.nim} (100%) rename doc/snippets/quickstart/{09/01/in.yaml => 08/00/01-in.yaml} (100%) rename doc/snippets/quickstart/{09/01 => 08/00}/title (100%) rename doc/snippets/quickstart/{09/02/code.nim => 08/01/00-code.nim} (100%) rename doc/snippets/quickstart/{09/02/in.yaml => 08/01/01-in.yaml} (100%) rename doc/snippets/quickstart/{09/02 => 08/01}/title (100%) delete mode 100644 doc/snippets/quickstart/09/title diff --git a/doc/rstPreproc.nim b/doc/rstPreproc.nim index 22522eb..92dfae4 100644 --- a/doc/rstPreproc.nim +++ b/doc/rstPreproc.nim @@ -85,21 +85,21 @@ proc outputExamples(curPath: string, level: int = 0) = case codeFiles.len of 0: discard of 1: - let (nullPath, name, extension) = codeFiles[0].splitFile() + let (_, _, extension) = codeFiles[0].splitFile() append(".. code:: " & extension[1..^1]) append(" :file: " & (curPath / codeFiles[0]) & '\l') of 2: append(".. raw:: html") append(" ") for codeFile in codeFiles: - append(" ") + append(" ") append(" \n
" & codeFile & "" & codeFile[3..^1] & "
\n") var first = true for codeFile in codeFiles: if first: first = false else: append(".. raw:: html\n \n") - let (nullPath, name, extension) = codeFile.splitFile() + let (_, _, extension) = codeFile.splitFile() append(".. code:: " & extension[1..^1]) append(" :file: " & (curPath / codeFile) & '\l') diff --git a/doc/snippets/quickstart/01/code.nim b/doc/snippets/quickstart/00/00-code.nim similarity index 100% rename from doc/snippets/quickstart/01/code.nim rename to doc/snippets/quickstart/00/00-code.nim diff --git a/doc/snippets/quickstart/01/out.yaml b/doc/snippets/quickstart/00/01-out.yaml similarity index 100% rename from doc/snippets/quickstart/01/out.yaml rename to doc/snippets/quickstart/00/01-out.yaml diff --git a/doc/snippets/quickstart/00/title b/doc/snippets/quickstart/00/title new file mode 100644 index 0000000..fd45668 --- /dev/null +++ b/doc/snippets/quickstart/00/title @@ -0,0 +1 @@ +Dumping Nim objects as YAML diff --git a/doc/snippets/quickstart/02/code.nim b/doc/snippets/quickstart/01/00-code.nim similarity index 100% rename from doc/snippets/quickstart/02/code.nim rename to doc/snippets/quickstart/01/00-code.nim diff --git a/doc/snippets/quickstart/02/in.yaml b/doc/snippets/quickstart/01/01-in.yaml similarity index 100% rename from doc/snippets/quickstart/02/in.yaml rename to doc/snippets/quickstart/01/01-in.yaml diff --git a/doc/snippets/quickstart/01/title b/doc/snippets/quickstart/01/title index fd45668..96a5007 100644 --- a/doc/snippets/quickstart/01/title +++ b/doc/snippets/quickstart/01/title @@ -1 +1 @@ -Dumping Nim objects as YAML +Loading Nim objects from YAML diff --git a/doc/snippets/quickstart/03/code.nim b/doc/snippets/quickstart/02/00-code.nim similarity index 100% rename from doc/snippets/quickstart/03/code.nim rename to doc/snippets/quickstart/02/00-code.nim diff --git a/doc/snippets/quickstart/03/out.yaml b/doc/snippets/quickstart/02/01-out.yaml similarity index 100% rename from doc/snippets/quickstart/03/out.yaml rename to doc/snippets/quickstart/02/01-out.yaml diff --git a/doc/snippets/quickstart/02/title b/doc/snippets/quickstart/02/title index 96a5007..944d448 100644 --- a/doc/snippets/quickstart/02/title +++ b/doc/snippets/quickstart/02/title @@ -1 +1 @@ -Loading Nim objects from YAML +Customizing output style diff --git a/doc/snippets/quickstart/04/code.nim b/doc/snippets/quickstart/03/00-code.nim similarity index 100% rename from doc/snippets/quickstart/04/code.nim rename to doc/snippets/quickstart/03/00-code.nim diff --git a/doc/snippets/quickstart/04/out.yaml b/doc/snippets/quickstart/03/01-out.yaml similarity index 100% rename from doc/snippets/quickstart/04/out.yaml rename to doc/snippets/quickstart/03/01-out.yaml diff --git a/doc/snippets/quickstart/03/title b/doc/snippets/quickstart/03/title index 944d448..4ffce07 100644 --- a/doc/snippets/quickstart/03/title +++ b/doc/snippets/quickstart/03/title @@ -1 +1 @@ -Customizing output style +Dumping reference types and cyclic structures diff --git a/doc/snippets/quickstart/05/code.nim b/doc/snippets/quickstart/04/00-code.nim similarity index 100% rename from doc/snippets/quickstart/05/code.nim rename to doc/snippets/quickstart/04/00-code.nim diff --git a/doc/snippets/quickstart/05/in.yaml b/doc/snippets/quickstart/04/01-in.yaml similarity index 100% rename from doc/snippets/quickstart/05/in.yaml rename to doc/snippets/quickstart/04/01-in.yaml diff --git a/doc/snippets/quickstart/04/title b/doc/snippets/quickstart/04/title index 4ffce07..225029d 100644 --- a/doc/snippets/quickstart/04/title +++ b/doc/snippets/quickstart/04/title @@ -1 +1 @@ -Dumping reference types and cyclic structures +Loading reference types and cyclic structures diff --git a/doc/snippets/quickstart/06/code.nim b/doc/snippets/quickstart/05/00-code.nim similarity index 100% rename from doc/snippets/quickstart/06/code.nim rename to doc/snippets/quickstart/05/00-code.nim diff --git a/doc/snippets/quickstart/06/out.yaml b/doc/snippets/quickstart/05/01-out.yaml similarity index 100% rename from doc/snippets/quickstart/06/out.yaml rename to doc/snippets/quickstart/05/01-out.yaml diff --git a/doc/snippets/quickstart/05/title b/doc/snippets/quickstart/05/title index 225029d..dda14d4 100644 --- a/doc/snippets/quickstart/05/title +++ b/doc/snippets/quickstart/05/title @@ -1 +1 @@ -Loading reference types and cyclic structures +Defining a custom tag uri for a type diff --git a/doc/snippets/quickstart/07/code.nim b/doc/snippets/quickstart/06/00-code.nim similarity index 100% rename from doc/snippets/quickstart/07/code.nim rename to doc/snippets/quickstart/06/00-code.nim diff --git a/doc/snippets/quickstart/07/out.yaml b/doc/snippets/quickstart/06/01-out.yaml similarity index 100% rename from doc/snippets/quickstart/07/out.yaml rename to doc/snippets/quickstart/06/01-out.yaml diff --git a/doc/snippets/quickstart/06/title b/doc/snippets/quickstart/06/title index dda14d4..21eb8cf 100644 --- a/doc/snippets/quickstart/06/title +++ b/doc/snippets/quickstart/06/title @@ -1 +1 @@ -Defining a custom tag uri for a type +Dumping Nim objects as JSON diff --git a/doc/snippets/quickstart/08/code.nim b/doc/snippets/quickstart/07/00-code.nim similarity index 100% rename from doc/snippets/quickstart/08/code.nim rename to doc/snippets/quickstart/07/00-code.nim diff --git a/doc/snippets/quickstart/08/in.yaml b/doc/snippets/quickstart/07/01-in.yaml similarity index 100% rename from doc/snippets/quickstart/08/in.yaml rename to doc/snippets/quickstart/07/01-in.yaml diff --git a/doc/snippets/quickstart/07/title b/doc/snippets/quickstart/07/title index 21eb8cf..5fb44c7 100644 --- a/doc/snippets/quickstart/07/title +++ b/doc/snippets/quickstart/07/title @@ -1 +1 @@ -Dumping Nim objects as JSON +Loading Nim objects from JSON diff --git a/doc/snippets/quickstart/09/01/code.nim b/doc/snippets/quickstart/08/00/00-code.nim similarity index 100% rename from doc/snippets/quickstart/09/01/code.nim rename to doc/snippets/quickstart/08/00/00-code.nim diff --git a/doc/snippets/quickstart/09/01/in.yaml b/doc/snippets/quickstart/08/00/01-in.yaml similarity index 100% rename from doc/snippets/quickstart/09/01/in.yaml rename to doc/snippets/quickstart/08/00/01-in.yaml diff --git a/doc/snippets/quickstart/09/01/title b/doc/snippets/quickstart/08/00/title similarity index 100% rename from doc/snippets/quickstart/09/01/title rename to doc/snippets/quickstart/08/00/title diff --git a/doc/snippets/quickstart/09/02/code.nim b/doc/snippets/quickstart/08/01/00-code.nim similarity index 100% rename from doc/snippets/quickstart/09/02/code.nim rename to doc/snippets/quickstart/08/01/00-code.nim diff --git a/doc/snippets/quickstart/09/02/in.yaml b/doc/snippets/quickstart/08/01/01-in.yaml similarity index 100% rename from doc/snippets/quickstart/09/02/in.yaml rename to doc/snippets/quickstart/08/01/01-in.yaml diff --git a/doc/snippets/quickstart/09/02/title b/doc/snippets/quickstart/08/01/title similarity index 100% rename from doc/snippets/quickstart/09/02/title rename to doc/snippets/quickstart/08/01/title diff --git a/doc/snippets/quickstart/08/title b/doc/snippets/quickstart/08/title index 5fb44c7..d413d92 100644 --- a/doc/snippets/quickstart/08/title +++ b/doc/snippets/quickstart/08/title @@ -1 +1 @@ -Loading Nim objects from JSON +Processing a Sequence of Heterogeneous Items diff --git a/doc/snippets/quickstart/09/title b/doc/snippets/quickstart/09/title deleted file mode 100644 index d413d92..0000000 --- a/doc/snippets/quickstart/09/title +++ /dev/null @@ -1 +0,0 @@ -Processing a Sequence of Heterogeneous Items From 0f2e077b871730602c41f0e1fc5157ce90756315 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sat, 8 Oct 2016 23:35:33 +0200 Subject: [PATCH 05/26] Added automatic test for quickstart snippets * There are errors which indicate real bugs! --- .gitignore | 2 + doc/snippets/quickstart/00/00-code.nim | 2 +- doc/snippets/quickstart/00/01-out.yaml | 8 +- doc/snippets/quickstart/01/00-code.nim | 2 +- doc/snippets/quickstart/02/00-code.nim | 6 +- doc/snippets/quickstart/02/01-out.yaml | 14 +-- doc/snippets/quickstart/03/00-code.nim | 2 +- doc/snippets/quickstart/03/01-out.yaml | 8 +- doc/snippets/quickstart/04/00-code.nim | 2 +- doc/snippets/quickstart/05/00-code.nim | 5 +- doc/snippets/quickstart/05/01-out.yaml | 4 +- doc/snippets/quickstart/06/00-code.nim | 2 +- doc/snippets/quickstart/06/01-out.yaml | 3 +- doc/snippets/quickstart/07/00-code.nim | 2 +- doc/snippets/quickstart/08/00/00-code.nim | 2 +- doc/snippets/quickstart/08/01/00-code.nim | 2 +- test/tquickstart.nim | 128 ++++++++++++++++++++++ yaml/serialization.nim | 3 + 18 files changed, 165 insertions(+), 32 deletions(-) create mode 100644 test/tquickstart.nim diff --git a/.gitignore b/.gitignore index a345654..704e960 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ test/tdom test/tserialization test/tjson test/yamlTestSuite +test/tquickstart test/*.exe test/*.pdb test/*.ilk @@ -19,4 +20,5 @@ bench/json docout doc/rstPreproc doc/tmp.rst +doc/**/code yaml-dev-kit diff --git a/doc/snippets/quickstart/00/00-code.nim b/doc/snippets/quickstart/00/00-code.nim index b1bec26..6354883 100644 --- a/doc/snippets/quickstart/00/00-code.nim +++ b/doc/snippets/quickstart/00/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml.serialization, streams type Person = object name : string age : int32 diff --git a/doc/snippets/quickstart/00/01-out.yaml b/doc/snippets/quickstart/00/01-out.yaml index 46d0583..207ff5c 100644 --- a/doc/snippets/quickstart/00/01-out.yaml +++ b/doc/snippets/quickstart/00/01-out.yaml @@ -1,8 +1,8 @@ %YAML 1.2 ---- !nim:system:seq(nim:custom:Person) -- +--- !nim:system:seq(nim:custom:Person) +- name: Karl Koch age: 23 -- +- name: Peter Pan - age: 12 \ No newline at end of file + age: 12 diff --git a/doc/snippets/quickstart/01/00-code.nim b/doc/snippets/quickstart/01/00-code.nim index 3ceb4ce..19bff02 100644 --- a/doc/snippets/quickstart/01/00-code.nim +++ b/doc/snippets/quickstart/01/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml.serialization, streams type Person = object name : string age : int32 diff --git a/doc/snippets/quickstart/02/00-code.nim b/doc/snippets/quickstart/02/00-code.nim index 2e80785..1743b07 100644 --- a/doc/snippets/quickstart/02/00-code.nim +++ b/doc/snippets/quickstart/02/00-code.nim @@ -1,13 +1,13 @@ -import yaml +import yaml.serialization, yaml.presenter, streams type Person = object name: string age: int32 -var personList: seq[Person] +var personList = newSeq[Person]() personList.add(Person(name: "Karl Koch", age: 23)) personList.add(Person(name: "Peter Pan", age: 12)) -var s = newFileStream("out.yaml") +var s = newFileStream("out.yaml", fmWrite) dump(personList, s, options = defineOptions( style = psCanonical, indentationStep = 3, diff --git a/doc/snippets/quickstart/02/01-out.yaml b/doc/snippets/quickstart/02/01-out.yaml index 76b4fc0..ac7ffcd 100644 --- a/doc/snippets/quickstart/02/01-out.yaml +++ b/doc/snippets/quickstart/02/01-out.yaml @@ -1,16 +1,16 @@ %YAML 1.1 ---- !nim:system:seq(nim:custom:Person) -[ - !nim:custom:Person { +--- +!nim:system:seq(nim:custom:Person) [ + !nim:custom:Person { ? !!str "name" : !!str "Karl Koch", ? !!str "age" : !nim:system:int32 "23" - }, - !nim:custom:Person { + }, + !nim:custom:Person { ? !!str "name" : !!str "Peter Pan", ? !!str "age" : !nim:system:int32 "12" - } -] \ No newline at end of file + } +] diff --git a/doc/snippets/quickstart/03/00-code.nim b/doc/snippets/quickstart/03/00-code.nim index 0e97b6c..e812eca 100644 --- a/doc/snippets/quickstart/03/00-code.nim +++ b/doc/snippets/quickstart/03/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml.serialization, streams type Node = ref NodeObj NodeObj = object diff --git a/doc/snippets/quickstart/03/01-out.yaml b/doc/snippets/quickstart/03/01-out.yaml index 25e569e..4762270 100644 --- a/doc/snippets/quickstart/03/01-out.yaml +++ b/doc/snippets/quickstart/03/01-out.yaml @@ -1,11 +1,11 @@ %YAML 1.2 ---- !nim:custom:NodeObj &a +--- !nim:custom:NodeObj &a name: Node 1 -left: +left: name: Node 2 left: !!null ~ - right: &b + right: &b name: Node 3 left: *a right: !!null ~ -right: *b \ No newline at end of file +right: *b diff --git a/doc/snippets/quickstart/04/00-code.nim b/doc/snippets/quickstart/04/00-code.nim index 983ae94..c698008 100644 --- a/doc/snippets/quickstart/04/00-code.nim +++ b/doc/snippets/quickstart/04/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml.serialization, streams type Node = ref NodeObj NodeObj = object diff --git a/doc/snippets/quickstart/05/00-code.nim b/doc/snippets/quickstart/05/00-code.nim index 411fb58..f2b7d9f 100644 --- a/doc/snippets/quickstart/05/00-code.nim +++ b/doc/snippets/quickstart/05/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml, streams type Mob = object level, experience: int32 drops: seq[string] @@ -9,6 +9,5 @@ setTagUri(seq[string], "!Drops") var mob = Mob(level: 42, experience: 1800, drops: @["Sword of Mob Slaying"]) var s = newFileStream("out.yaml", fmWrite) -dump(mob, s, - options = defineOptions(tagStyle = tsAll)) +dump(mob, s, tagStyle = tsAll) s.close() \ No newline at end of file diff --git a/doc/snippets/quickstart/05/01-out.yaml b/doc/snippets/quickstart/05/01-out.yaml index cf6deba..b3c155a 100644 --- a/doc/snippets/quickstart/05/01-out.yaml +++ b/doc/snippets/quickstart/05/01-out.yaml @@ -1,5 +1,5 @@ %YAML 1.2 ---- !Mob +--- !Mob !nim:field level: !nim:system:int32 42 !nim:field experience: !nim:system:int32 1800 -!nim:field drops: !Drops [!!str Sword of Mob Slaying] \ No newline at end of file +!nim:field drops: !Drops [!!str Sword of Mob Slaying] diff --git a/doc/snippets/quickstart/06/00-code.nim b/doc/snippets/quickstart/06/00-code.nim index 7b91553..babe9f3 100644 --- a/doc/snippets/quickstart/06/00-code.nim +++ b/doc/snippets/quickstart/06/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml.serialization, yaml.presenter, streams type Person = object name : string age : int32 diff --git a/doc/snippets/quickstart/06/01-out.yaml b/doc/snippets/quickstart/06/01-out.yaml index 3be727c..06befc2 100644 --- a/doc/snippets/quickstart/06/01-out.yaml +++ b/doc/snippets/quickstart/06/01-out.yaml @@ -1,3 +1,4 @@ + [ { "name": "Karl Koch", @@ -7,4 +8,4 @@ "name": "Peter Pan", "age": 12 } -] \ No newline at end of file +] diff --git a/doc/snippets/quickstart/07/00-code.nim b/doc/snippets/quickstart/07/00-code.nim index cf86f03..1312ee5 100644 --- a/doc/snippets/quickstart/07/00-code.nim +++ b/doc/snippets/quickstart/07/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml.serialization, streams type Person = object name : string age : int32 diff --git a/doc/snippets/quickstart/08/00/00-code.nim b/doc/snippets/quickstart/08/00/00-code.nim index 323c410..4bf5947 100644 --- a/doc/snippets/quickstart/08/00/00-code.nim +++ b/doc/snippets/quickstart/08/00/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml, streams type Person = object name: string diff --git a/doc/snippets/quickstart/08/01/00-code.nim b/doc/snippets/quickstart/08/01/00-code.nim index e1ada99..59b0f84 100644 --- a/doc/snippets/quickstart/08/01/00-code.nim +++ b/doc/snippets/quickstart/08/01/00-code.nim @@ -1,4 +1,4 @@ -import yaml +import yaml, streams type Person = object name: string diff --git a/test/tquickstart.nim b/test/tquickstart.nim new file mode 100644 index 0000000..fd88c43 --- /dev/null +++ b/test/tquickstart.nim @@ -0,0 +1,128 @@ +import unittest, os, osproc, macros, strutils, streams + +proc inputTest(path: string): bool = + let + inFileOrig = path / "01-in.yaml" + inFileDest = path / "in.yaml" + codeFileOrig = path / "00-code.nim" + codeFileDest = path / "code.nim" + exeFileDest = when defined(windows): path / "code.exe" else: path / "code" + currentDir = getCurrentDir() + basePath = currentDir / ".." + absolutePath = currentDir / path + copyFile(inFileOrig, inFileDest) + copyFile(codeFileOrig, codeFileDest) + defer: + removeFile(inFileDest) + removeFile(codeFileDest) + var process = startProcess("nim c --hints:off -p:" & escape(basePath) & + " code.nim", path, [], nil, {poStdErrToStdOut, poEvalCommand}) + setCurrentDir(currentDir) # workaround for https://github.com/nim-lang/Nim/issues/4867 + defer: + process.close() + if process.waitForExit() != 0: + echo "compiler output:" + echo "================\n" + echo process.outputStream().readAll() + result = false + else: + defer: removeFile(exeFileDest) + process.close() + process = startProcess(absolutePath / "code", absolutePath, [], nil, + {poStdErrToStdOut, poEvalCommand}) + setCurrentDir(currentDir) # workaround for https://github.com/nim-lang/Nim/issues/4867 + if process.waitForExit() != 0: + echo "executable output:" + echo "==================\n" + echo process.outputStream().readAll() + result = false + else: result = true + +proc outputTest(path: string): bool = + let + codeFileOrig = path / "00-code.nim" + codeFileDest = path / "code.nim" + exeFileDest = when defined(windows): path / "code.exe" else: path / "code" + currentDir = getCurrentDir() + basePath = currentDir / ".." + absolutePath = currentDir / path + copyFile(codeFileOrig, codeFileDest) + defer: removeFile(codeFileDest) + var process = startProcess("nim c --hints:off -p:" & escape(basePath) & + " code.nim", path, [], nil, {poStdErrToStdOut, poEvalCommand}) + defer: process.close() + setCurrentDir(currentDir) # workaround for https://github.com/nim-lang/Nim/issues/4867 + if process.waitForExit() != 0: + echo "compiler output:" + echo "================\n" + echo process.outputStream().readAll() + result = false + else: + defer: removeFile(exeFileDest) + process.close() + process = startProcess(absolutePath / "code", absolutePath, [], nil, + {poStdErrToStdOut, poEvalCommand}) + setCurrentDir(currentDir) # workaround for https://github.com/nim-lang/Nim/issues/4867 + if process.waitForExit() != 0: + echo "executable output:" + echo "==================\n" + echo process.outputStream().readAll() + result = false + else: + var + expected = open(path / "01-out.yaml", fmRead) + actual = open(path / "out.yaml", fmRead) + lineNumber = 1 + defer: + expected.close() + actual.close() + var + expectedLine = "" + actualLine = "" + while true: + if expected.readLine(expectedLine): + if actual.readLine(actualLine): + if expectedLine != actualLine: + echo "difference at line #", lineNumber, ':' + echo "expected: ", escape(expectedLine) + echo " actual: ", escape(actualLine) + return false + else: + echo "actual output has fewer lines than expected; ", + "first missing line: #", lineNumber + echo "expected: ", escape(expectedLine) + return false + else: + if actual.readLine(actualLine): + echo "actual output has more lines than expected; ", + "first unexpected line: #", lineNumber + echo "content: ", escape(actualLine) + return false + else: break + lineNumber.inc() + result = true + +proc testsFor(path: string, root: bool = true, titlePrefix: string = ""): + NimNode {.compileTime.} = + result = newStmtList() + let title = titlePrefix & slurp(path / "title").splitLines()[0] + if fileExists(path / "00-code.nim"): + var test = newCall("test", newLit(title)) + if fileExists(path / "01-in.yaml"): + test.add(newCall("doAssert", newCall("inputTest", newLit(path)))) + elif fileExists(path / "01-out.yaml"): + test.add(newCall("doAssert", newCall("outputTest", newLit(path)))) + else: + echo "Error: neither 01-in.yaml nor 01-out.yaml exists in " & path & '!' + quit 1 + result.add(test) + for kind, childPath in walkDir(path): + if kind == pcDir: + if childPath != path / "nimcache": + result.add(testsFor(childPath, false, if root: "" else: title & ' ')) + if root: + result = newCall("suite", newLit(title), result) + +macro genTests(): untyped = testsFor("../doc/snippets/quickstart") + +genTests() \ No newline at end of file diff --git a/yaml/serialization.nim b/yaml/serialization.nim index df95ac5..756b52e 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -18,6 +18,9 @@ import tables, typetraits, strutils, macros, streams import parser, taglib, presenter, stream, ../private/internal, hints +export stream + # *something* in here needs externally visible `==`(x,y: AnchorId), + # but I cannot figure out what. binding it would be the better option. type SerializationContext* = ref object From 4bde3a79867796cfa7edcdfd95c051deadf06de7 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sun, 9 Oct 2016 11:48:52 +0200 Subject: [PATCH 06/26] Fixed issues in quickstart and taglib * Updated expected YAML in 02 * Fixed duplicate TagIds in taglib --- doc/snippets/quickstart/02/01-out.yaml | 8 ++++---- test/tquickstart.nim | 7 +++++-- yaml/taglib.nim | 9 +++++++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/doc/snippets/quickstart/02/01-out.yaml b/doc/snippets/quickstart/02/01-out.yaml index ac7ffcd..89d97a6 100644 --- a/doc/snippets/quickstart/02/01-out.yaml +++ b/doc/snippets/quickstart/02/01-out.yaml @@ -2,15 +2,15 @@ --- !nim:system:seq(nim:custom:Person) [ !nim:custom:Person { - ? !!str "name" + ? !nim:field "name" : !!str "Karl Koch", - ? !!str "age" + ? !nim:field "age" : !nim:system:int32 "23" }, !nim:custom:Person { - ? !!str "name" + ? !nim:field "name" : !!str "Peter Pan", - ? !!str "age" + ? !nim:field "age" : !nim:system:int32 "12" } ] diff --git a/test/tquickstart.nim b/test/tquickstart.nim index fd88c43..d33c1ba 100644 --- a/test/tquickstart.nim +++ b/test/tquickstart.nim @@ -46,6 +46,8 @@ proc outputTest(path: string): bool = currentDir = getCurrentDir() basePath = currentDir / ".." absolutePath = currentDir / path + outFileExpected = path / "01-out.yaml" + outFileActual = path / "out.yaml" copyFile(codeFileOrig, codeFileDest) defer: removeFile(codeFileDest) var process = startProcess("nim c --hints:off -p:" & escape(basePath) & @@ -69,9 +71,10 @@ proc outputTest(path: string): bool = echo process.outputStream().readAll() result = false else: + defer: removeFile(outFileActual) var - expected = open(path / "01-out.yaml", fmRead) - actual = open(path / "out.yaml", fmRead) + expected = open(outFileExpected, fmRead) + actual = open(outFileActual, fmRead) lineNumber = 1 defer: expected.close() diff --git a/yaml/taglib.nim b/yaml/taglib.nim index 25f99e8..e077ef8 100644 --- a/yaml/taglib.nim +++ b/yaml/taglib.nim @@ -94,7 +94,10 @@ const ## for seqs that are nil. This tag is used regardless of the seq's generic ## type parameter. - yFirstCustomTagId* : TagId = 1000.TagId ## \ + yFirstStaticTagId* : TagId = 1000.TagId ## \ + ## The first ``TagId`` assigned by the ``setTagId`` templates. + + yFirstCustomTagId* : TagId = 10000.TagId ## \ ## The first ``TagId`` which should be assigned to an URI that does not ## exist in the ``YamlTagLibrary`` which is used for parsing. @@ -217,7 +220,7 @@ var ## `serializable <#serializable,stmt,stmt>`_. var - nextStaticTagId {.compileTime.} = 100.TagId ## \ + nextStaticTagId {.compileTime.} = yFirstStaticTagId ## \ ## used for generating unique TagIds with ``setTagUri``. registeredUris {.compileTime.} = newSeq[string]() ## \ ## Since Table doesn't really work at compile time, we also store @@ -249,6 +252,8 @@ template setTagUri*(t: typedesc, uri: string, idName: untyped): typed = static: registeredUris.add(uri) nextStaticTagId = TagId(int(nextStaticTagId) + 1) + when nextStaticTagId == yFirstCustomTagId: + {.fatal: "Too many tags!".} serializationTagLibrary.tags[uri] = idName proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName ## autogenerated From 846b836e9217dc64693554d588c9356c6b73f388 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sun, 9 Oct 2016 15:05:18 +0200 Subject: [PATCH 07/26] Integrated parser and quickstart tests * Now executed when calling `nim tests` * Refactored parser tests to use unittest * Fixed some pwd issues in tests --- README.md | 7 +-- config.nims | 16 +++++-- test/tests.nim | 2 +- test/tparser.nim | 106 +++++++++++++++++++++++++++++++++++++++++ test/tquickstart.nim | 54 ++++++++++----------- test/yamlTestSuite.nim | 83 -------------------------------- 6 files changed, 150 insertions(+), 118 deletions(-) create mode 100644 test/tparser.nim delete mode 100644 test/yamlTestSuite.nim diff --git a/README.md b/README.md index af0dd5d..8a421ae 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,16 @@ available as tags in this repository and can be fetched via nimble: ## Developers ```bash -nim tests # runs unit tests (serialization, dom, json) - # for parser tests, see yamlTestSuite +nim tests # runs all tests +nim lexerTests # run lexer tests +nim parserTests # run parser tests (git-clones yaml-dev-kit) nim serializationTests # runs serialization tests +nim quickstartTests # run tests for quickstart snippets from documentation nim documentation # builds documentation to folder docout nim server # builds the REST server used for the testing ground nim bench # runs benchmarks, requires libyaml nim clean # guess nim build # build a library -nim yamlTestSuite # execute YAML test suite (git-clones yaml-dev-kit) ``` NimYAML needs at least Nim 0.15.0 in order to compile. diff --git a/config.nims b/config.nims index efa86f8..5340dbe 100644 --- a/config.nims +++ b/config.nims @@ -8,15 +8,25 @@ task tests, "Run all tests": --verbosity:0 setCommand "c", "test/tests" -task yamlTestSuite, "Run YAML 1.2 test suite": +task lexerTests, "Run lexer tests": --r --verbosity:0 - setCommand "c", "test/yamlTestSuite" + setCommand "c", "test/tlex" + +task parserTests, "Run parser tests": + --r + --verbosity:0 + setCommand "c", "test/tparser" task serializationTests, "Run serialization tests": --r --verbosity:0 - setCommand "c", "test/serializing" + setCommand "c", "test/tserialization" + +task quickstartTests, "Run quickstart tests": + --r + --verbosity:0 + setCommand "c", "test/tquickstart" task documentation, "Generate documentation": exec "mkdir -p docout" diff --git a/test/tests.nim b/test/tests.nim index 26231ad..4be6a76 100644 --- a/test/tests.nim +++ b/test/tests.nim @@ -4,4 +4,4 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. -import tlex, tjson, tserialization, tdom \ No newline at end of file +import tlex, tjson, tserialization, tdom, tparser, tquickstart \ No newline at end of file diff --git a/test/tparser.nim b/test/tparser.nim new file mode 100644 index 0000000..9da5fa2 --- /dev/null +++ b/test/tparser.nim @@ -0,0 +1,106 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +import os, osproc, terminal, strutils, streams, macros, unittest +import testEventParser, commonTestUtils +import "../yaml" + +const devKitFolder = "yaml-dev-kit" + +proc echoError(msg: string) = + styledWriteLine(stdout, fgRed, "[error] ", fgWhite, msg, resetStyle) + +proc ensureDevKitCloneCorrect() {.compileTime.} = + if dirExists(devKitFolder): + var isCorrectClone = true + if dirExists(devKitFolder / ".git"): + let remoteUrl = + staticExec("cd " & devKitFolder & " && git remote get-url origin") + if remoteUrl != "https://github.com/ingydotnet/yaml-dev-kit.git": + isCorrectClone = false + let branches = staticExec("cd " & devKitFolder & " && git branch") + if "* data" notin branches.splitLines(): + isCorrectClone = false + if isCorrectClone: + let updateOutput = staticExec("git pull") + #if uError != 0: + # echo "could not update yaml-dev-kit! please fix this problem and compile again." + # echo "output:\n" + # echo "$ git pull" + # echo updateOutput + # quit 1 + else: + echo devKitFolder, " exists, but is not in expected state. Make sure it is a git repo," + echo "cloned from https://github.com/ingydotnet/yaml-dev-kit.git, and the data branch" + echo "is active. Alternatively, delete the folder " & devKitFolder & '.' + quit 1 + else: + let cloneOutput = staticExec("git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data") + #if cError != 0: + # echo "could not clone https://github.com/ingydotnet/yaml-dev-kit.git. Make sure" + # echo "you are connected to the internet and your proxy settings are correct. output:\n" + # echo "$ git clone https://github.com/ingydotnet/yaml-dev-kit.git" + # echo cloneOutput + # quit 1 + +proc parserTest(path: string): bool = + var + tagLib = initExtendedTagLibrary() + parser = newYamlParser(tagLib) + actualIn = newFileStream(path / "in.yaml") + actual = parser.parse(actualIn) + expectedIn = newFileStream(path / "test.event") + expected = parseEventStream(expectedIn, tagLib) + defer: + actualIn.close() + expectedIn.close() + var i = 1 + try: + while not actual.finished(): + let actualEvent = actual.next() + if expected.finished(): + echoError("At token #" & $i & ": Expected stream end, got " & + $actualEvent.kind) + return false + let expectedEvent = expected.next() + if expectedEvent != actualEvent: + printDifference(expectedEvent, actualEvent) + echoError("At token #" & $i & + ": Actual tokens do not match expected tokens") + return false + i.inc() + if not expected.finished(): + echoError("Got fewer tokens than expected, first missing " & + "token: " & $expected.next().kind) + return false + except: + let e = getCurrentException() + if e.parent of YamlParserError: + let pe = (ref YamlParserError)(e.parent) + echo "line ", pe.line, ", column ", pe.column, ": ", pe.msg + echo pe.lineContent + else: echo e.msg + echoError("Catched an exception at token #" & $i & + " test was not successful") + return false + result = true + +macro genTests(): untyped = + ensureDevKitCloneCorrect() + result = newStmtList() + let pwd = staticExec("pwd") + for kind, dirName in walkDir(devKitFolder, true): + if kind == pcDir: + if dirName in [".git", "name", "tags", "meta"]: continue + # see https://github.com/nim-lang/Nim/issues/4871 + let title = slurp(pwd / devKitFolder / dirName / "===") + result.add(newCall("test", + newLit(strip(title) & " [" & + dirName & ']'), newCall("doAssert", newCall("parserTest", + newLit(devKitFolder / dirName))))) + result = newCall("suite", newLit("Parser Tests (from yaml-dev-kit)"), result) + +genTests() \ No newline at end of file diff --git a/test/tquickstart.nim b/test/tquickstart.nim index d33c1ba..018fc71 100644 --- a/test/tquickstart.nim +++ b/test/tquickstart.nim @@ -1,23 +1,21 @@ import unittest, os, osproc, macros, strutils, streams -proc inputTest(path: string): bool = +proc inputTest(basePath, path: string): bool = let - inFileOrig = path / "01-in.yaml" - inFileDest = path / "in.yaml" - codeFileOrig = path / "00-code.nim" - codeFileDest = path / "code.nim" - exeFileDest = when defined(windows): path / "code.exe" else: path / "code" - currentDir = getCurrentDir() - basePath = currentDir / ".." - absolutePath = currentDir / path + absolutePath = basePath / path + inFileOrig = absolutePath / "01-in.yaml" + inFileDest = absolutePath / "in.yaml" + codeFileOrig = absolutePath / "00-code.nim" + codeFileDest = absolutePath / "code.nim" + exeFileDest = when defined(windows): absolutePath / "code.exe" else: + absolutePath / "code" copyFile(inFileOrig, inFileDest) copyFile(codeFileOrig, codeFileDest) defer: removeFile(inFileDest) removeFile(codeFileDest) var process = startProcess("nim c --hints:off -p:" & escape(basePath) & - " code.nim", path, [], nil, {poStdErrToStdOut, poEvalCommand}) - setCurrentDir(currentDir) # workaround for https://github.com/nim-lang/Nim/issues/4867 + " code.nim", absolutePath, [], nil, {poStdErrToStdOut, poEvalCommand}) defer: process.close() if process.waitForExit() != 0: @@ -30,7 +28,6 @@ proc inputTest(path: string): bool = process.close() process = startProcess(absolutePath / "code", absolutePath, [], nil, {poStdErrToStdOut, poEvalCommand}) - setCurrentDir(currentDir) # workaround for https://github.com/nim-lang/Nim/issues/4867 if process.waitForExit() != 0: echo "executable output:" echo "==================\n" @@ -38,22 +35,20 @@ proc inputTest(path: string): bool = result = false else: result = true -proc outputTest(path: string): bool = +proc outputTest(basePath, path: string): bool = let - codeFileOrig = path / "00-code.nim" - codeFileDest = path / "code.nim" - exeFileDest = when defined(windows): path / "code.exe" else: path / "code" - currentDir = getCurrentDir() - basePath = currentDir / ".." - absolutePath = currentDir / path - outFileExpected = path / "01-out.yaml" - outFileActual = path / "out.yaml" + absolutePath = basePath / path + codeFileOrig = absolutePath / "00-code.nim" + codeFileDest = absolutePath / "code.nim" + exeFileDest = when defined(windows): absolutePath / "code.exe" else: + absolutePath / "code" + outFileExpected = absolutePath / "01-out.yaml" + outFileActual = absolutePath / "out.yaml" copyFile(codeFileOrig, codeFileDest) defer: removeFile(codeFileDest) var process = startProcess("nim c --hints:off -p:" & escape(basePath) & - " code.nim", path, [], nil, {poStdErrToStdOut, poEvalCommand}) + " code.nim", absolutePath, [], nil, {poStdErrToStdOut, poEvalCommand}) defer: process.close() - setCurrentDir(currentDir) # workaround for https://github.com/nim-lang/Nim/issues/4867 if process.waitForExit() != 0: echo "compiler output:" echo "================\n" @@ -64,7 +59,6 @@ proc outputTest(path: string): bool = process.close() process = startProcess(absolutePath / "code", absolutePath, [], nil, {poStdErrToStdOut, poEvalCommand}) - setCurrentDir(currentDir) # workaround for https://github.com/nim-lang/Nim/issues/4867 if process.waitForExit() != 0: echo "executable output:" echo "==================\n" @@ -108,13 +102,17 @@ proc outputTest(path: string): bool = proc testsFor(path: string, root: bool = true, titlePrefix: string = ""): NimNode {.compileTime.} = result = newStmtList() - let title = titlePrefix & slurp(path / "title").splitLines()[0] + let + baseDir = staticExec("pwd") + title = titlePrefix & slurp(baseDir / path / "title").splitLines()[0] if fileExists(path / "00-code.nim"): var test = newCall("test", newLit(title)) if fileExists(path / "01-in.yaml"): - test.add(newCall("doAssert", newCall("inputTest", newLit(path)))) + test.add(newCall("doAssert", newCall("inputTest", newLit(baseDir), + newLit(path)))) elif fileExists(path / "01-out.yaml"): - test.add(newCall("doAssert", newCall("outputTest", newLit(path)))) + test.add(newCall("doAssert", newCall("outputTest", newLit(baseDir), + newLit(path)))) else: echo "Error: neither 01-in.yaml nor 01-out.yaml exists in " & path & '!' quit 1 @@ -126,6 +124,6 @@ proc testsFor(path: string, root: bool = true, titlePrefix: string = ""): if root: result = newCall("suite", newLit(title), result) -macro genTests(): untyped = testsFor("../doc/snippets/quickstart") +macro genTests(): untyped = testsFor("doc/snippets/quickstart") genTests() \ No newline at end of file diff --git a/test/yamlTestSuite.nim b/test/yamlTestSuite.nim deleted file mode 100644 index 2423af4..0000000 --- a/test/yamlTestSuite.nim +++ /dev/null @@ -1,83 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2016 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -import os, terminal, strutils, streams -import testEventParser, commonTestUtils -import "../yaml" - -const gitCmd = - "git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data" - -proc echoError(msg: string) = - styledWriteLine(stdout, fgRed, "[error] ", fgWhite, msg, resetStyle) - -proc echoInfo(msg: string) = - styledWriteLine(stdout, fgGreen, "[info] ", fgWhite, msg, resetStyle) - -removeDir("yaml-dev-kit") -if execShellCmd(gitCmd) != 0: - echoError("Could not check out yaml-dev-kit (no internet connection?)") - quit(1) - -var gotErrors = false - -for kind, dirPath in walkDir("yaml-dev-kit"): - block curTest: - if kind == pcDir: - if dirPath[^4..^1] in [".git", "name", "tags", "meta"]: continue - var - tagLib = initExtendedTagLibrary() - parser = newYamlParser(tagLib) - actualIn = newFileStream(dirPath / "in.yaml") - actual = parser.parse(actualIn) - expectedIn = newFileStream(dirPath / "test.event") - expected = parseEventStream(expectedIn, tagLib) - styledWriteLine(stdout, fgBlue, "[test] ", fgWhite, dirPath[^4..^1], - ": ", strip(readFile(dirPath / "===")), resetStyle) - var i = 1 - try: - while not actual.finished(): - let actualEvent = actual.next() - if expected.finished(): - echoError("At token #" & $i & ": Expected stream end, got " & - $actualEvent.kind) - gotErrors = true - actualIn.close() - expectedIn.close() - break curTest - let expectedEvent = expected.next() - if expectedEvent != actualEvent: - printDifference(expectedEvent, actualEvent) - echoError("At token #" & $i & - ": Actual tokens do not match expected tokens") - gotErrors = true - actualIn.close() - expectedIn.close() - break curTest - i.inc() - if not expected.finished(): - echoError("Got fewer tokens than expected, first missing " & - "token: " & $expected.next().kind) - gotErrors = true - except: - gotErrors = true - let e = getCurrentException() - if e.parent of YamlParserError: - let pe = (ref YamlParserError)(e.parent) - echo "line ", pe.line, ", column ", pe.column, ": ", pe.msg - echo pe.lineContent - else: echo e.msg - echoError("Catched an exception at token #" & $i & - " test was not successful") - actualIn.close() - expectedIn.close() - -if gotErrors: - echoError("There were errors while running the tests") - quit(1) -else: - echoInfo("All tests were successful") - quit(0) \ No newline at end of file From d2e5c9e0a9b482ee522c454c40931090dbe6fd45 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sun, 9 Oct 2016 15:21:27 +0200 Subject: [PATCH 08/26] Hopefully fixes travis build --- .travis.yml | 3 +-- test/tparser.nim | 11 ++++++----- test/tquickstart.nim | 13 +++++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7ab07b5..fa7a49f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,8 +37,7 @@ install: before_script: - export PATH="nim-$BRANCH/bin${PATH:+:$PATH}" script: - - nim tests --verbosity:0 - - nim yamlTestSuite --verbosity:0 + - nim tests cache: directories: - nim-master diff --git a/test/tparser.nim b/test/tparser.nim index 9da5fa2..81966d1 100644 --- a/test/tparser.nim +++ b/test/tparser.nim @@ -40,11 +40,12 @@ proc ensureDevKitCloneCorrect() {.compileTime.} = else: let cloneOutput = staticExec("git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data") #if cError != 0: - # echo "could not clone https://github.com/ingydotnet/yaml-dev-kit.git. Make sure" - # echo "you are connected to the internet and your proxy settings are correct. output:\n" - # echo "$ git clone https://github.com/ingydotnet/yaml-dev-kit.git" - # echo cloneOutput - # quit 1 + if not dirExists(devKitFolder): + echo "could not clone https://github.com/ingydotnet/yaml-dev-kit.git. Make sure" + echo "you are connected to the internet and your proxy settings are correct. output:\n" + echo "$ git clone https://github.com/ingydotnet/yaml-dev-kit.git" + echo cloneOutput + quit 1 proc parserTest(path: string): bool = var diff --git a/test/tquickstart.nim b/test/tquickstart.nim index 018fc71..d80a2b4 100644 --- a/test/tquickstart.nim +++ b/test/tquickstart.nim @@ -1,6 +1,6 @@ import unittest, os, osproc, macros, strutils, streams -proc inputTest(basePath, path: string): bool = +proc inputTest(basePath, path, nimPath: string): bool = let absolutePath = basePath / path inFileOrig = absolutePath / "01-in.yaml" @@ -14,7 +14,7 @@ proc inputTest(basePath, path: string): bool = defer: removeFile(inFileDest) removeFile(codeFileDest) - var process = startProcess("nim c --hints:off -p:" & escape(basePath) & + var process = startProcess(nimPath & " c --hints:off -p:" & escape(basePath) & " code.nim", absolutePath, [], nil, {poStdErrToStdOut, poEvalCommand}) defer: process.close() @@ -35,7 +35,7 @@ proc inputTest(basePath, path: string): bool = result = false else: result = true -proc outputTest(basePath, path: string): bool = +proc outputTest(basePath, path, nimPath: string): bool = let absolutePath = basePath / path codeFileOrig = absolutePath / "00-code.nim" @@ -46,7 +46,7 @@ proc outputTest(basePath, path: string): bool = outFileActual = absolutePath / "out.yaml" copyFile(codeFileOrig, codeFileDest) defer: removeFile(codeFileDest) - var process = startProcess("nim c --hints:off -p:" & escape(basePath) & + var process = startProcess(nimPath & " c --hints:off -p:" & escape(basePath) & " code.nim", absolutePath, [], nil, {poStdErrToStdOut, poEvalCommand}) defer: process.close() if process.waitForExit() != 0: @@ -104,15 +104,16 @@ proc testsFor(path: string, root: bool = true, titlePrefix: string = ""): result = newStmtList() let baseDir = staticExec("pwd") + nimPath = staticExec("which nim") title = titlePrefix & slurp(baseDir / path / "title").splitLines()[0] if fileExists(path / "00-code.nim"): var test = newCall("test", newLit(title)) if fileExists(path / "01-in.yaml"): test.add(newCall("doAssert", newCall("inputTest", newLit(baseDir), - newLit(path)))) + newLit(path), newLit(nimPath)))) elif fileExists(path / "01-out.yaml"): test.add(newCall("doAssert", newCall("outputTest", newLit(baseDir), - newLit(path)))) + newLit(path), newLit(nimPath)))) else: echo "Error: neither 01-in.yaml nor 01-out.yaml exists in " & path & '!' quit 1 From e1f3ac9b0770f434609ecb22ed0ae21b306da041 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sun, 9 Oct 2016 15:30:50 +0200 Subject: [PATCH 09/26] Another try to fix travis build --- test/tparser.nim | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/test/tparser.nim b/test/tparser.nim index 81966d1..b58c398 100644 --- a/test/tparser.nim +++ b/test/tparser.nim @@ -13,19 +13,20 @@ const devKitFolder = "yaml-dev-kit" proc echoError(msg: string) = styledWriteLine(stdout, fgRed, "[error] ", fgWhite, msg, resetStyle) -proc ensureDevKitCloneCorrect() {.compileTime.} = - if dirExists(devKitFolder): +proc ensureDevKitCloneCorrect(pwd: string) {.compileTime.} = + let absolutePath = pwd / devKitFolder + if dirExists(absolutePath): var isCorrectClone = true - if dirExists(devKitFolder / ".git"): + if dirExists(absolutePath / ".git"): let remoteUrl = - staticExec("cd " & devKitFolder & " && git remote get-url origin") + staticExec("cd \"" & absolutePath & "\" && git remote get-url origin") if remoteUrl != "https://github.com/ingydotnet/yaml-dev-kit.git": isCorrectClone = false - let branches = staticExec("cd " & devKitFolder & " && git branch") + let branches = staticExec("cd \"" & absolutePath & "\" && git branch") if "* data" notin branches.splitLines(): isCorrectClone = false if isCorrectClone: - let updateOutput = staticExec("git pull") + let updateOutput = staticExec("cd \"" & absolutePath & "\" && git pull") #if uError != 0: # echo "could not update yaml-dev-kit! please fix this problem and compile again." # echo "output:\n" @@ -38,9 +39,10 @@ proc ensureDevKitCloneCorrect() {.compileTime.} = echo "is active. Alternatively, delete the folder " & devKitFolder & '.' quit 1 else: - let cloneOutput = staticExec("git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data") + let cloneOutput = staticExec("cd \"" & pwd & + "\" && git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data") #if cError != 0: - if not dirExists(devKitFolder): + if not dirExists(absolutePath): echo "could not clone https://github.com/ingydotnet/yaml-dev-kit.git. Make sure" echo "you are connected to the internet and your proxy settings are correct. output:\n" echo "$ git clone https://github.com/ingydotnet/yaml-dev-kit.git" @@ -90,18 +92,21 @@ proc parserTest(path: string): bool = result = true macro genTests(): untyped = - ensureDevKitCloneCorrect() + let + pwd = staticExec("pwd") + absolutePath = pwd / devKitFolder + echo "[tparser] Generating tests from " & escape(absolutePath) + ensureDevKitCloneCorrect(pwd) result = newStmtList() - let pwd = staticExec("pwd") - for kind, dirName in walkDir(devKitFolder, true): + for kind, dirName in walkDir(absolutePath, true): if kind == pcDir: if dirName in [".git", "name", "tags", "meta"]: continue - # see https://github.com/nim-lang/Nim/issues/4871 - let title = slurp(pwd / devKitFolder / dirName / "===") + echo "[tparser] Test: " & dirName + let title = slurp(absolutePath / dirName / "===") result.add(newCall("test", newLit(strip(title) & " [" & dirName & ']'), newCall("doAssert", newCall("parserTest", - newLit(devKitFolder / dirName))))) + newLit(absolutePath / dirName))))) result = newCall("suite", newLit("Parser Tests (from yaml-dev-kit)"), result) genTests() \ No newline at end of file From 636ee8129594e9c78ea75bba1bdd3acec49218fb Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sun, 9 Oct 2016 15:37:20 +0200 Subject: [PATCH 10/26] Still working on travis --- test/tparser.nim | 3 ++- test/tquickstart.nim | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/tparser.nim b/test/tparser.nim index b58c398..4b3a124 100644 --- a/test/tparser.nim +++ b/test/tparser.nim @@ -42,7 +42,8 @@ proc ensureDevKitCloneCorrect(pwd: string) {.compileTime.} = let cloneOutput = staticExec("cd \"" & pwd & "\" && git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data") #if cError != 0: - if not dirExists(absolutePath): + if not(dirExists(absolutePath)) or not(dirExists(absolutePath / ".git")) or + not(dirExists(absolutePath / "229Q")): echo "could not clone https://github.com/ingydotnet/yaml-dev-kit.git. Make sure" echo "you are connected to the internet and your proxy settings are correct. output:\n" echo "$ git clone https://github.com/ingydotnet/yaml-dev-kit.git" diff --git a/test/tquickstart.nim b/test/tquickstart.nim index d80a2b4..5c8180a 100644 --- a/test/tquickstart.nim +++ b/test/tquickstart.nim @@ -104,7 +104,8 @@ proc testsFor(path: string, root: bool = true, titlePrefix: string = ""): result = newStmtList() let baseDir = staticExec("pwd") - nimPath = staticExec("which nim") + nimPathRaw = staticExec("which nim") + nimPath = if nimPathRaw[0] == '/': nimPathRaw else: baseDir / nimPathRaw title = titlePrefix & slurp(baseDir / path / "title").splitLines()[0] if fileExists(path / "00-code.nim"): var test = newCall("test", newLit(title)) From ea90b4d2dbf51f791dde0fa6ee258ca576da2623 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sun, 9 Oct 2016 15:42:43 +0200 Subject: [PATCH 11/26] Debugging remaining travis issues --- test/tparser.nim | 1 + 1 file changed, 1 insertion(+) diff --git a/test/tparser.nim b/test/tparser.nim index 4b3a124..382d68c 100644 --- a/test/tparser.nim +++ b/test/tparser.nim @@ -100,6 +100,7 @@ macro genTests(): untyped = ensureDevKitCloneCorrect(pwd) result = newStmtList() for kind, dirName in walkDir(absolutePath, true): + echo "[tparser] directory item: kind=", kind, ", name=", dirName if kind == pcDir: if dirName in [".git", "name", "tags", "meta"]: continue echo "[tparser] Test: " & dirName From 98c16afb47704c4647d865f7ca2775c0f680868c Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sun, 9 Oct 2016 15:53:47 +0200 Subject: [PATCH 12/26] This might finally fix travis --- test/tparser.nim | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/test/tparser.nim b/test/tparser.nim index 382d68c..9ace20b 100644 --- a/test/tparser.nim +++ b/test/tparser.nim @@ -95,20 +95,19 @@ proc parserTest(path: string): bool = macro genTests(): untyped = let pwd = staticExec("pwd") - absolutePath = pwd / devKitFolder - echo "[tparser] Generating tests from " & escape(absolutePath) + absolutePath = '"' & (pwd / devKitFolder) & '"' + echo "[tparser] Generating tests from " & absolutePath ensureDevKitCloneCorrect(pwd) result = newStmtList() - for kind, dirName in walkDir(absolutePath, true): - echo "[tparser] directory item: kind=", kind, ", name=", dirName - if kind == pcDir: - if dirName in [".git", "name", "tags", "meta"]: continue - echo "[tparser] Test: " & dirName - let title = slurp(absolutePath / dirName / "===") - result.add(newCall("test", - newLit(strip(title) & " [" & - dirName & ']'), newCall("doAssert", newCall("parserTest", - newLit(absolutePath / dirName))))) + # walkDir for some crude reason does not work with travis build + let dirItems = staticExec("ls -1d " & absolutePath / "*") + for dirPath in dirItems.splitLines(): + if dirPath[^4..^1] in [".git", "name", "tags", "meta"]: continue + let title = slurp(dirPath / "===") + result.add(newCall("test", + newLit(strip(title) & " [" & + dirPath[^4..^1] & ']'), newCall("doAssert", newCall("parserTest", + newLit(dirPath))))) result = newCall("suite", newLit("Parser Tests (from yaml-dev-kit)"), result) genTests() \ No newline at end of file From fa14a119579b2b60fa2a89ea0a1ea8875c5bfd66 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sun, 9 Oct 2016 16:10:11 +0200 Subject: [PATCH 13/26] Separated test suites for travis --- .travis.yml | 7 ++++++- config.nims | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fa7a49f..44b4883 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,12 @@ install: before_script: - export PATH="nim-$BRANCH/bin${PATH:+:$PATH}" script: - - nim tests + - nim lexerTests + - nim parserTests + - nim jsonTests + - nim domTests + - nim serializationTests + - nim quickstartTests cache: directories: - nim-master diff --git a/config.nims b/config.nims index 5340dbe..57572d2 100644 --- a/config.nims +++ b/config.nims @@ -18,6 +18,16 @@ task parserTests, "Run parser tests": --verbosity:0 setCommand "c", "test/tparser" +task jsonTests, "Run JSON tests": + --r + --verbosity:0 + setCommand "c", "test/tjson" + +task domTests, "Run DOM tests": + --r + --verbosity:0 + setCommand "c", "test/tdom" + task serializationTests, "Run serialization tests": --r --verbosity:0 From 29352fa4fe1a9a91f300a11179f362049c786b3c Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Mon, 10 Oct 2016 20:16:54 +0200 Subject: [PATCH 14/26] Use global tag handle for NimYAML * NimYAML now uses the tag prefix tag:nimyaml.org,2016: * That tag handle is shortened to !n! when presenting * Also fixed some minor bugs dealing with tag handles --- doc/snippets/quickstart/00/01-out.yaml | 3 +- doc/snippets/quickstart/02/01-out.yaml | 19 ++-- doc/snippets/quickstart/03/01-out.yaml | 3 +- doc/snippets/quickstart/04/01-in.yaml | 7 +- doc/snippets/quickstart/05/01-out.yaml | 7 +- doc/snippets/quickstart/08/00/00-code.nim | 2 +- doc/snippets/quickstart/08/00/01-in.yaml | 3 +- doc/snippets/quickstart/08/01/00-code.nim | 2 +- doc/snippets/quickstart/08/01/01-in.yaml | 3 +- test/tserialization.nim | 74 ++++++------ yaml/presenter.nim | 21 ++-- yaml/serialization.nim | 35 +++--- yaml/taglib.nim | 131 ++++++++++++++-------- 13 files changed, 181 insertions(+), 129 deletions(-) diff --git a/doc/snippets/quickstart/00/01-out.yaml b/doc/snippets/quickstart/00/01-out.yaml index 207ff5c..ae95158 100644 --- a/doc/snippets/quickstart/00/01-out.yaml +++ b/doc/snippets/quickstart/00/01-out.yaml @@ -1,5 +1,6 @@ %YAML 1.2 ---- !nim:system:seq(nim:custom:Person) +%TAG !n! tag:nimyaml.org,2016: +--- !n!system:seq(tag:nimyaml.org;2016:custom:Person) - name: Karl Koch age: 23 diff --git a/doc/snippets/quickstart/02/01-out.yaml b/doc/snippets/quickstart/02/01-out.yaml index 89d97a6..c2919ff 100644 --- a/doc/snippets/quickstart/02/01-out.yaml +++ b/doc/snippets/quickstart/02/01-out.yaml @@ -1,16 +1,17 @@ %YAML 1.1 +%TAG !n! tag:nimyaml.org,2016: --- -!nim:system:seq(nim:custom:Person) [ - !nim:custom:Person { - ? !nim:field "name" +!n!system:seq(tag:nimyaml.org;2016:custom:Person) [ + !n!custom:Person { + ? !n!field "name" : !!str "Karl Koch", - ? !nim:field "age" - : !nim:system:int32 "23" + ? !n!field "age" + : !n!system:int32 "23" }, - !nim:custom:Person { - ? !nim:field "name" + !n!custom:Person { + ? !n!field "name" : !!str "Peter Pan", - ? !nim:field "age" - : !nim:system:int32 "12" + ? !n!field "age" + : !n!system:int32 "12" } ] diff --git a/doc/snippets/quickstart/03/01-out.yaml b/doc/snippets/quickstart/03/01-out.yaml index 4762270..ee3adbd 100644 --- a/doc/snippets/quickstart/03/01-out.yaml +++ b/doc/snippets/quickstart/03/01-out.yaml @@ -1,5 +1,6 @@ %YAML 1.2 ---- !nim:custom:NodeObj &a +%TAG !n! tag:nimyaml.org,2016: +--- !n!custom:NodeObj &a name: Node 1 left: name: Node 2 diff --git a/doc/snippets/quickstart/04/01-in.yaml b/doc/snippets/quickstart/04/01-in.yaml index 78d163b..9c99c16 100644 --- a/doc/snippets/quickstart/04/01-in.yaml +++ b/doc/snippets/quickstart/04/01-in.yaml @@ -1,11 +1,12 @@ %YAML 1.2 ---- !nim:custom:NodeObj &a +%TAG !n! tag:nimyaml.org,2016: +--- !n!custom:NodeObj &a name: Node 1 left: name: Node 2 left: ~ - right: &b + right: &b name: Node 3 left: *a right: ~ -right: *b \ No newline at end of file +right: *b diff --git a/doc/snippets/quickstart/05/01-out.yaml b/doc/snippets/quickstart/05/01-out.yaml index b3c155a..16710fc 100644 --- a/doc/snippets/quickstart/05/01-out.yaml +++ b/doc/snippets/quickstart/05/01-out.yaml @@ -1,5 +1,6 @@ %YAML 1.2 +%TAG !n! tag:nimyaml.org,2016: --- !Mob -!nim:field level: !nim:system:int32 42 -!nim:field experience: !nim:system:int32 1800 -!nim:field drops: !Drops [!!str Sword of Mob Slaying] +!n!field level: !n!system:int32 42 +!n!field experience: !n!system:int32 1800 +!n!field drops: !Drops [!!str Sword of Mob Slaying] diff --git a/doc/snippets/quickstart/08/00/00-code.nim b/doc/snippets/quickstart/08/00/00-code.nim index 4bf5947..7be0d02 100644 --- a/doc/snippets/quickstart/08/00/00-code.nim +++ b/doc/snippets/quickstart/08/00/00-code.nim @@ -19,7 +19,7 @@ type of ckNone: discard -setTagUri(Person, "!nim:demo:Person") +setTagUri(Person, nimTag("demo:Person")) # tell NimYAML to use Container as implicit type. # only possible with variant object types where diff --git a/doc/snippets/quickstart/08/00/01-in.yaml b/doc/snippets/quickstart/08/00/01-in.yaml index aae04fc..7923272 100644 --- a/doc/snippets/quickstart/08/00/01-in.yaml +++ b/doc/snippets/quickstart/08/00/01-in.yaml @@ -1,8 +1,9 @@ %YAML 1.2 +%TAG !n! tag:nimyaml.org,2016: --- - this is a string - 42 - false - !!str 23 -- !nim:demo:Person {name: Trillian} +- !n!demo:Person {name: Trillian} - !!null \ No newline at end of file diff --git a/doc/snippets/quickstart/08/01/00-code.nim b/doc/snippets/quickstart/08/01/00-code.nim index 59b0f84..15046ee 100644 --- a/doc/snippets/quickstart/08/01/00-code.nim +++ b/doc/snippets/quickstart/08/01/00-code.nim @@ -2,7 +2,7 @@ import yaml, streams type Person = object name: string -setTagUri(Person, "!nim:demo:Person", yTagPerson) +setTagUri(Person, nimTag("demo:Person"), yTagPerson) var s = newFileStream("in.yaml", fmRead) diff --git a/doc/snippets/quickstart/08/01/01-in.yaml b/doc/snippets/quickstart/08/01/01-in.yaml index 8164e19..10fff1b 100644 --- a/doc/snippets/quickstart/08/01/01-in.yaml +++ b/doc/snippets/quickstart/08/01/01-in.yaml @@ -1,7 +1,8 @@ %YAML 1.2 +%TAG !n! tag:nimyaml.org,2016: --- !!seq - this is a string - 42 - false - !!str 23 -- !nim:demo:Person {name: Trillian} \ No newline at end of file +- !n!demo:Person {name: Trillian} \ No newline at end of file diff --git a/test/tserialization.nim b/test/tserialization.nim index 861dbc0..4432626 100644 --- a/test/tserialization.nim +++ b/test/tserialization.nim @@ -45,6 +45,8 @@ setTagUri(TrafficLight, "!tl") setTagUri(Node, "!example.net:Node") setTagUri(BetterInt, "!test:BetterInt") +const yamlDirs = "%YAML 1.2\n%TAG !n! tag:nimyaml.org,2016:\n--- " + proc representObject*(value: BetterInt, ts: TagStyle = tsNone, c: SerializationContext, tag: TagId) {.raises: [].} = var @@ -104,7 +106,7 @@ suite "Serialization": test "Dump integer without fixed length": var input = -4247 var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n\"-4247\"", output + assertStringEqual yamlDirs & "\n\"-4247\"", output when sizeof(int) == sizeof(int64): input = int(int32.high) + 1 @@ -162,7 +164,7 @@ suite "Serialization": assert(result == 14) test "Load nil string": - let input = newStringStream("!nim:nil:string \"\"") + let input = newStringStream("! \"\"") var result: string load(input, result) assert isNil(result) @@ -170,7 +172,7 @@ suite "Serialization": test "Dump nil string": let input: string = nil var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n!nim:nil:string \"\"", output + assertStringEqual yamlDirs & "\n!n!nil:string \"\"", output test "Load string sequence": let input = newStringStream(" - a\n - b") @@ -183,10 +185,10 @@ suite "Serialization": test "Dump string sequence": var input = @["a", "b"] var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output + assertStringEqual yamlDirs & "\n- a\n- b", output test "Load nil seq": - let input = newStringStream("!nim:nil:seq \"\"") + let input = newStringStream("! \"\"") var result: seq[int] load(input, result) assert isNil(result) @@ -194,7 +196,7 @@ suite "Serialization": test "Dump nil seq": let input: seq[int] = nil var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n!nim:nil:seq \"\"", output + assertStringEqual yamlDirs & "\n!n!nil:seq \"\"", output test "Load char set": let input = newStringStream("- a\n- b") @@ -207,7 +209,7 @@ suite "Serialization": test "Dump char set": var input = {'a', 'b'} var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output + assertStringEqual yamlDirs & "\n- a\n- b", output test "Load array": let input = newStringStream("- 23\n- 42\n- 47") @@ -220,7 +222,7 @@ suite "Serialization": test "Dump array": let input = [23'i32, 42'i32, 47'i32] var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n- 23\n- 42\n- 47", output + assertStringEqual yamlDirs & "\n- 23\n- 42\n- 47", output test "Load Table[int, string]": let input = newStringStream("23: dreiundzwanzig\n42: zweiundvierzig") @@ -235,7 +237,7 @@ suite "Serialization": input[23] = "dreiundzwanzig" input[42] = "zweiundvierzig" var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual("%YAML 1.2\n--- \n23: dreiundzwanzig\n42: zweiundvierzig", + assertStringEqual(yamlDirs & "\n23: dreiundzwanzig\n42: zweiundvierzig", output) test "Load OrderedTable[tuple[int32, int32], string]": @@ -259,8 +261,8 @@ suite "Serialization": input.add((a: 23'i32, b: 42'i32), "dreiundzwanzigzweiundvierzig") input.add((a: 13'i32, b: 47'i32), "dreizehnsiebenundvierzig") var output = dump(input, tsRootOnly, asTidy, blockOnly) - assertStringEqual("""%YAML 1.2 ---- !nim:tables:OrderedTable(nim:tuple(nim:system:int32,nim:system:int32),tag:yaml.org,2002:str) + assertStringEqual(yamlDirs & + """!n!tables:OrderedTable(tag:nimyaml.org;2016:tuple(tag:nimyaml.org;2016:system:int32;tag:nimyaml.org;2016:system:int32);tag:yaml.org;2002:str) - ? a: 23 @@ -284,11 +286,11 @@ suite "Serialization": test "Dump Sequences in Sequence": let input = @[@[1.int32, 2.int32, 3.int32], @[4.int32, 5.int32], @[6.int32]] var output = dump(input, tsNone) - assertStringEqual "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]", output + assertStringEqual yamlDirs & "\n- [1, 2, 3]\n- [4, 5]\n- [6]", output test "Load Enum": let input = - newStringStream("!nim:system:seq(tl)\n- !tl tlRed\n- tlGreen\n- tlYellow") + newStringStream("!\n- !tl tlRed\n- tlGreen\n- tlYellow") var result: seq[TrafficLight] load(input, result) assert result.len == 3 @@ -299,7 +301,7 @@ suite "Serialization": test "Dump Enum": let input = @[tlRed, tlGreen, tlYellow] var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n- tlRed\n- tlGreen\n- tlYellow", output + assertStringEqual yamlDirs & "\n- tlRed\n- tlGreen\n- tlYellow", output test "Load Tuple": let input = newStringStream("str: value\ni: 42\nb: true") @@ -312,7 +314,7 @@ suite "Serialization": test "Dump Tuple": let input = (str: "value", i: 42.int32, b: true) var output = dump(input, tsNone) - assertStringEqual "%YAML 1.2\n--- \nstr: value\ni: 42\nb: y", output + assertStringEqual yamlDirs & "\nstr: value\ni: 42\nb: y", output test "Load Tuple - unknown field": let input = "str: value\nfoo: bar\ni: 42\nb: true" @@ -358,8 +360,8 @@ suite "Serialization": test "Dump custom object": let input = Person(firstnamechar: 'P', surname: "Pan", age: 12) var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual( - "%YAML 1.2\n--- \nfirstnamechar: P\nsurname: Pan\nage: 12", output) + assertStringEqual(yamlDirs & + "\nfirstnamechar: P\nsurname: Pan\nage: 12", output) test "Load custom object - unknown field": let input = " firstnamechar: P\n surname: Pan\n age: 12\n occupation: free" @@ -380,8 +382,8 @@ suite "Serialization": load(input, result) test "Load sequence with explicit tags": - let input = newStringStream("--- !nim:system:seq(" & - "tag:yaml.org,2002:str)\n- !!str one\n- !!str two") + let input = newStringStream(yamlDirs & "!n!system:seq(" & + "tag:yaml.org;2002:str)\n- !!str one\n- !!str two") var result: seq[string] load(input, result) assert result[0] == "one" @@ -390,12 +392,12 @@ suite "Serialization": test "Dump sequence with explicit tags": let input = @["one", "two"] var output = dump(input, tsAll, asTidy, blockOnly) - assertStringEqual("%YAML 1.2\n--- !nim:system:seq(" & - "tag:yaml.org,2002:str) \n- !!str one\n- !!str two", output) + assertStringEqual(yamlDirs & "!n!system:seq(" & + "tag:yaml.org;2002:str) \n- !!str one\n- !!str two", output) test "Load custom object with explicit root tag": let input = newStringStream( - "--- !nim:custom:Person\nfirstnamechar: P\nsurname: Pan\nage: 12") + "--- !\nfirstnamechar: P\nsurname: Pan\nage: 12") var result: Person load(input, result) assert result.firstnamechar == 'P' @@ -405,9 +407,8 @@ suite "Serialization": test "Dump custom object with explicit root tag": let input = Person(firstnamechar: 'P', surname: "Pan", age: 12) var output = dump(input, tsRootOnly, asTidy, blockOnly) - assertStringEqual("%YAML 1.2\n" & - "--- !nim:custom:Person \nfirstnamechar: P\nsurname: Pan\nage: 12", - output) + assertStringEqual(yamlDirs & + "!n!custom:Person \nfirstnamechar: P\nsurname: Pan\nage: 12", output) test "Load custom variant object": let input = newStringStream( @@ -427,8 +428,8 @@ suite "Serialization": let input = @[Animal(name: "Bastet", kind: akCat, purringIntensity: 7), Animal(name: "Anubis", kind: akDog, barkometer: 13)] var output = dump(input, tsNone, asTidy, blockOnly) - assertStringEqual """%YAML 1.2 ---- + assertStringEqual yamlDirs & """ + - - name: Bastet @@ -459,8 +460,7 @@ suite "Serialization": b.next = c c.next = a var output = dump(a, tsRootOnly, asTidy, blockOnly) - assertStringEqual """%YAML 1.2 ---- !example.net:Node &a + assertStringEqual yamlDirs & """!example.net:Node &a value: a next: value: b @@ -469,13 +469,12 @@ next: next: *a""", output test "Load cyclic data structure": - let input = newStringStream("""%YAML 1.2 ---- !nim:system:seq(example.net:Node) -- &a + let input = newStringStream(yamlDirs & """!n!system:seq(example.net:Node) +- &a value: a - next: &b + next: &b value: b - next: &c + next: &c value: c next: *a - *b @@ -517,8 +516,8 @@ next: input.add(new string) input[1][] = "~" var output = dump(input, tsRootOnly, asTidy, blockOnly) - assertStringEqual( - "%YAML 1.2\n--- !nim:system:seq(tag:yaml.org,2002:str) \n- !!null ~\n- !!str ~", + assertStringEqual(yamlDirs & + "!n!system:seq(tag:yaml.org;2002:str) \n- !!null ~\n- !!str ~", output) test "Custom constructObject": @@ -532,8 +531,7 @@ next: test "Custom representObject": let input = @[1.BetterInt, 9998887.BetterInt, 98312.BetterInt] var output = dump(input, tsAll, asTidy, blockOnly) - assertStringEqual """%YAML 1.2 ---- !nim:system:seq(test:BetterInt) + assertStringEqual yamlDirs & """!n!system:seq(test:BetterInt) - !test:BetterInt 1 - !test:BetterInt 9_998_887 - !test:BetterInt 98_312""", output diff --git a/yaml/presenter.nim b/yaml/presenter.nim index f5b9537..9dd1151 100644 --- a/yaml/presenter.nim +++ b/yaml/presenter.nim @@ -413,12 +413,10 @@ proc writeTagAndAnchor(target: PresenterTarget, tag: TagId, try: if tag notin [yTagQuestionMark, yTagExclamationMark]: let tagUri = tagLib.uri(tag) - if tagUri.startsWith(tagLib.secondaryPrefix): - target.append("!!") - target.append(tagUri[18..tagUri.high]) - target.append(' ') - elif tagUri.startsWith("!"): - target.append(tagUri) + let (handle, length) = tagLib.searchHandle(tagUri) + if length > 0: + target.append(handle) + target.append(tagUri[length..tagUri.high]) target.append(' ') else: target.append("!<") @@ -461,8 +459,15 @@ proc doPresent(s: var YamlStream, target: PresenterTarget, of ov1_2: target.append("%YAML 1.2" & newline) of ov1_1: target.append("%YAML 1.1" & newLine) of ovNone: discard - if tagLib.secondaryPrefix != yamlTagRepositoryPrefix: - target.append("%TAG !! " & tagLib.secondaryPrefix & newline) + for prefix, handle in tagLib.handles(): + if handle == "!": + if prefix != "!": + target.append("%TAG ! " & prefix & newline) + elif handle == "!!": + if prefix != yamlTagRepositoryPrefix: + target.append("%TAG !! " & prefix & newline) + else: + target.append("%TAG " & handle & ' ' & prefix & newline) target.append("--- ") except: var e = newException(YamlPresenterOutputError, "") diff --git a/yaml/serialization.nim b/yaml/serialization.nim index 756b52e..52cdcb1 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -104,9 +104,17 @@ proc lazyLoadTag(uri: string): TagId {.inline, raises: [].} = proc safeTagUri(id: TagId): string {.raises: [].} = try: - let uri = serializationTagLibrary.uri(id) - if uri.len > 0 and uri[0] == '!': return uri[1..uri.len - 1] - else: return uri + var + uri = serializationTagLibrary.uri(id) + i = 0 + # '!' is not allowed inside a tag handle + if uri.len > 0 and uri[0] == '!': uri = uri[1..^1] + # ',' is not allowed after a tag handle in the suffix because it's a flow + # indicator + for c in uri.mitems(): + if c == ',': c = ';' + inc(i) + return uri except KeyError: internalError("Unexpected KeyError for TagId " & $id) proc constructionError(s: YamlStream, msg: string): ref YamlConstructionError = @@ -314,11 +322,11 @@ proc representObject*(value: char, ts: TagStyle, c: SerializationContext, c.put(scalarEvent("" & value, tag, yAnchorNone)) proc yamlTag*[I](T: typedesc[seq[I]]): TagId {.inline, raises: [].} = - let uri = "!nim:system:seq(" & safeTagUri(yamlTag(I)) & ')' + let uri = nimTag("system:seq(" & safeTagUri(yamlTag(I)) & ')') result = lazyLoadTag(uri) proc yamlTag*[I](T: typedesc[set[I]]): TagId {.inline, raises: [].} = - let uri = "!nim:system:set(" & safeTagUri(yamlTag(I)) & ')' + let uri = nimTag("system:set(" & safeTagUri(yamlTag(I)) & ')') result = lazyLoadTag(uri) proc constructObject*[T](s: var YamlStream, c: ConstructionContext, @@ -360,8 +368,8 @@ proc representObject*[T](value: seq[T]|set[T], ts: TagStyle, proc yamlTag*[I, V](T: typedesc[array[I, V]]): TagId {.inline, raises: [].} = const rangeName = name(I) - let uri = "!nim:system:array(" & rangeName[6..rangeName.high()] & "," & - safeTagUri(yamlTag(V)) & ')' + let uri = nimTag("system:array(" & rangeName[6..rangeName.high()] & ';' & + safeTagUri(yamlTag(V)) & ')') result = lazyLoadTag(uri) proc constructObject*[I, T](s: var YamlStream, c: ConstructionContext, @@ -391,8 +399,8 @@ proc representObject*[I, T](value: array[I, T], ts: TagStyle, proc yamlTag*[K, V](T: typedesc[Table[K, V]]): TagId {.inline, raises: [].} = try: - let uri = "!nim:tables:Table(" & safeTagUri(yamlTag(K)) & "," & - safeTagUri(yamlTag(V)) & ")" + let uri = nimTag("tables:Table(" & safeTagUri(yamlTag(K)) & ';' & + safeTagUri(yamlTag(V)) & ")") result = lazyLoadTag(uri) except KeyError: # cannot happen (theoretically, you know) @@ -430,8 +438,8 @@ proc representObject*[K, V](value: Table[K, V], ts: TagStyle, proc yamlTag*[K, V](T: typedesc[OrderedTable[K, V]]): TagId {.inline, raises: [].} = try: - let uri = "!nim:tables:OrderedTable(" & safeTagUri(yamlTag(K)) & "," & - safeTagUri(yamlTag(V)) & ")" + let uri = nimTag("tables:OrderedTable(" & safeTagUri(yamlTag(K)) & ';' & + safeTagUri(yamlTag(V)) & ")") result = lazyLoadTag(uri) except KeyError: # cannot happen (theoretically, you know) @@ -475,7 +483,7 @@ proc representObject*[K, V](value: OrderedTable[K, V], ts: TagStyle, proc yamlTag*(T: typedesc[object|enum]): TagId {.inline, raises: [].} = - var uri = "!nim:custom:" & (typetraits.name(type(T))) + var uri = nimTag("custom:" & (typetraits.name(type(T)))) try: serializationTagLibrary.tags[uri] except KeyError: serializationTagLibrary.registerUri(uri) @@ -483,7 +491,7 @@ proc yamlTag*(T: typedesc[tuple]): TagId {.inline, raises: [].} = var i: T - uri = "!nim:tuple(" + uri = nimTag("tuple(") first = true for name, value in fieldPairs(i): if first: first = false @@ -741,6 +749,7 @@ macro constructImplicitVariantObject(s, c, r, possibleTagIds: untyped, )) ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkTryStmt).add( newStmtList(raiseStmt), newNimNode(nnkExceptBranch).add( + newIdentNode("KeyError"), newNimNode(nnkDiscardStmt).add(newEmptyNode()) )))) result = newStmtList(newCall("reset", r), ifStmt) diff --git a/yaml/taglib.nim b/yaml/taglib.nim index e077ef8..1d34232 100644 --- a/yaml/taglib.nim +++ b/yaml/taglib.nim @@ -12,7 +12,7 @@ ## and create own tags. It also enables you to define tags for types used with ## the serialization API. -import tables, macros, hashes +import tables, macros, hashes, strutils type TagId* = distinct int ## \ @@ -41,7 +41,7 @@ type ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_. tags*: Table[string, TagId] nextCustomTagId*: TagId - secondaryPrefix*: string + tagHandles: Table[string, string] const # failsafe schema @@ -102,6 +102,7 @@ const ## exist in the ``YamlTagLibrary`` which is used for parsing. yamlTagRepositoryPrefix* = "tag:yaml.org,2002:" + nimyamlTagRepositoryPrefix* = "tag:nimyaml.org,2016:" proc `==`*(left, right: TagId): bool {.borrow.} proc hash*(id: TagId): Hash {.borrow.} @@ -133,7 +134,7 @@ proc initTagLibrary*(): TagLibrary {.raises: [].} = ## ``yFirstCustomTagId``. new(result) result.tags = initTable[string, TagId]() - result.secondaryPrefix = yamlTagRepositoryPrefix + result.tagHandles = {"!": "!", yamlTagRepositoryPrefix : "!!"}.toTable() result.nextCustomTagId = yFirstCustomTagId proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} = @@ -149,6 +150,9 @@ proc uri*(tagLib: TagLibrary, id: TagId): string {.raises: [KeyError].} = if iId == id: return iUri raise newException(KeyError, "Unknown tag id: " & $id) +template y(suffix: string): string = yamlTagRepositoryPrefix & suffix +template n(suffix: string): string = nimyamlTagRepositoryPrefix & suffix + proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} = ## Contains only: ## - ``!`` @@ -159,9 +163,9 @@ proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} = result = initTagLibrary() result.tags["!"] = yTagExclamationMark result.tags["?"] = yTagQuestionMark - result.tags["tag:yaml.org,2002:str"] = yTagString - result.tags["tag:yaml.org,2002:seq"] = yTagSequence - result.tags["tag:yaml.org,2002:map"] = yTagMapping + result.tags[y"str"] = yTagString + result.tags[y"seq"] = yTagSequence + result.tags[y"map"] = yTagMapping proc initCoreTagLibrary*(): TagLibrary {.raises: [].} = ## Contains everything in ``initFailsafeTagLibrary`` plus: @@ -170,10 +174,10 @@ proc initCoreTagLibrary*(): TagLibrary {.raises: [].} = ## - ``!!int`` ## - ``!!float`` result = initFailsafeTagLibrary() - result.tags["tag:yaml.org,2002:null"] = yTagNull - result.tags["tag:yaml.org,2002:bool"] = yTagBoolean - result.tags["tag:yaml.org,2002:int"] = yTagInteger - result.tags["tag:yaml.org,2002:float"] = yTagFloat + result.tags[y"null"] = yTagNull + result.tags[y"bool"] = yTagBoolean + result.tags[y"int"] = yTagInteger + result.tags[y"float"] = yTagFloat proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} = ## Contains everything from ``initCoreTagLibrary`` plus: @@ -186,29 +190,29 @@ proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} = ## - ``!!value`` ## - ``!!yaml`` result = initCoreTagLibrary() - result.tags["tag:yaml.org,2002:omap"] = yTagOrderedMap - result.tags["tag:yaml.org,2002:pairs"] = yTagPairs - result.tags["tag:yaml.org,2002:binary"] = yTagBinary - result.tags["tag:yaml.org,2002:merge"] = yTagMerge - result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp - result.tags["tag:yaml.org,2002:value"] = yTagValue - result.tags["tag:yaml.org,2002:yaml"] = yTagYaml - + result.tags[y"omap"] = yTagOrderedMap + result.tags[y"pairs"] = yTagPairs + result.tags[y"binary"] = yTagBinary + result.tags[y"merge"] = yTagMerge + result.tags[y"timestamp"] = yTagTimestamp + result.tags[y"value"] = yTagValue + result.tags[y"yaml"] = yTagYaml proc initSerializationTagLibrary*(): TagLibrary = result = initTagLibrary() + result.tagHandles[nimyamlTagRepositoryPrefix] = "!n!" result.tags["!"] = yTagExclamationMark result.tags["?"] = yTagQuestionMark - result.tags["tag:yaml.org,2002:str"] = yTagString - result.tags["tag:yaml.org,2002:null"] = yTagNull - result.tags["tag:yaml.org,2002:bool"] = yTagBoolean - result.tags["tag:yaml.org,2002:float"] = yTagFloat - result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp - result.tags["tag:yaml.org,2002:value"] = yTagValue - result.tags["tag:yaml.org,2002:binary"] = yTagBinary - result.tags["!nim:field"] = yTagNimField - result.tags["!nim:nil:string"] = yTagNimNilString - result.tags["!nim:nil:seq"] = yTagNimNilSeq + result.tags[y"str"] = yTagString + result.tags[y"null"] = yTagNull + result.tags[y"bool"] = yTagBoolean + result.tags[y"float"] = yTagFloat + result.tags[y"timestamp"] = yTagTimestamp + result.tags[y"value"] = yTagValue + result.tags[y"binary"] = yTagBinary + result.tags[n"field"] = yTagNimField + result.tags[n"nil:string"] = yTagNimNilString + result.tags[n"nil:seq"] = yTagNimNilSeq var serializationTagLibrary* = initSerializationTagLibrary() ## \ @@ -262,27 +266,56 @@ static: # standard YAML tags used by serialization registeredUris.add("!") registeredUris.add("?") - registeredUris.add("tag:yaml.org,2002:str") - registeredUris.add("tag:yaml.org,2002:null") - registeredUris.add("tag:yaml.org,2002:bool") - registeredUris.add("tag:yaml.org,2002:float") - registeredUris.add("tag:yaml.org,2002:timestamp") - registeredUris.add("tag:yaml.org,2002:value") - registeredUris.add("tag:yaml.org,2002:binary") + registeredUris.add(y"str") + registeredUris.add(y"null") + registeredUris.add(y"bool") + registeredUris.add(y"float") + registeredUris.add(y"timestamp") + registeredUris.add(y"value") + registeredUris.add(y"binary") # special tags used by serialization - registeredUris.add("!nim:field") - registeredUris.add("!nim:nil:string") - registeredUris.add("!nim:nil:seq") + registeredUris.add(n"field") + registeredUris.add(n"nil:string") + registeredUris.add(n"nil:seq") # tags for Nim's standard types -setTagUri(char, "!nim:system:char", yTagNimChar) -setTagUri(int8, "!nim:system:int8", yTagNimInt8) -setTagUri(int16, "!nim:system:int16", yTagNimInt16) -setTagUri(int32, "!nim:system:int32", yTagNimInt32) -setTagUri(int64, "!nim:system:int64", yTagNimInt64) -setTagUri(uint8, "!nim:system:uint8", yTagNimUInt8) -setTagUri(uint16, "!nim:system:uint16", yTagNimUInt16) -setTagUri(uint32, "!nim:system:uint32", yTagNimUInt32) -setTagUri(uint64, "!nim:system:uint64", yTagNimUInt64) -setTagUri(float32, "!nim:system:float32", yTagNimFloat32) -setTagUri(float64, "!nim:system:float64", yTagNimFloat64) \ No newline at end of file +setTagUri(char, n"system:char", yTagNimChar) +setTagUri(int8, n"system:int8", yTagNimInt8) +setTagUri(int16, n"system:int16", yTagNimInt16) +setTagUri(int32, n"system:int32", yTagNimInt32) +setTagUri(int64, n"system:int64", yTagNimInt64) +setTagUri(uint8, n"system:uint8", yTagNimUInt8) +setTagUri(uint16, n"system:uint16", yTagNimUInt16) +setTagUri(uint32, n"system:uint32", yTagNimUInt32) +setTagUri(uint64, n"system:uint64", yTagNimUInt64) +setTagUri(float32, n"system:float32", yTagNimFloat32) +setTagUri(float64, n"system:float64", yTagNimFloat64) + +proc registerHandle*(tagLib: TagLibrary, handle, prefix: string) = + ## Registers a handle for a prefix. When presenting any tag that starts with + ## this prefix, the handle is used instead. Also causes the presenter to + ## output a TAG directive for the handle. + taglib.tagHandles[prefix] = handle + +proc searchHandle*(tagLib: TagLibrary, tag: string): + tuple[handle: string, len: int] {.raises: [].} = + ## search in the registered tag handles for one whose prefix matches the start + ## of the given tag. If multiple registered handles match, the one with the + ## longest prefix is returned. If no registered handle matches, (nil, 0) is + ## returned. + result.len = 0 + for key, value in tagLib.tagHandles: + if key.len > result.len: + if tag.startsWith(key): + result.len = key.len + result.handle = value + +iterator handles*(tagLib: TagLibrary): tuple[prefix, handle: string] = + ## iterate over registered tag handles that may be used as shortcuts + ## (e.g. ``!n!`` for ``tag:nimyaml.org,2016:``) + for key, value in tagLib.tagHandles: yield (key, value) + +proc nimTag*(suffix: string): string = + ## prepends NimYAML's tag repository prefix to the given suffix. For example, + ## ``nimTag("system:char")`` yields ``"tag:nimyaml.org,2016:system:char"``. + nimyamlTagRepositoryPrefix & suffix \ No newline at end of file From b3d83025f70e85d4e23d38621cfc8ff4a6119566 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Wed, 12 Oct 2016 22:39:59 +0200 Subject: [PATCH 15/26] Started transient implementation * represent of simple object works * TODO: variant objects * TODO: construct --- yaml/serialization.nim | 87 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 9 deletions(-) diff --git a/yaml/serialization.nim b/yaml/serialization.nim index 52cdcb1..5debffb 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -622,6 +622,12 @@ proc isVariantObject(t: typedesc): bool {.compileTime.} = if child.kind == nnkRecCase: return true return false +let + implicitVariantObjectMarker {.compileTime.} = + newIdentNode(":implicitVariantObject") + transientBitvectorProc {.compileTime.} = + newIdentNode(":transientObject") + proc constructObject*[O: object|tuple]( s: var YamlStream, c: ConstructionContext, result: var O) {.raises: [YamlConstructionError, YamlStreamError].} = @@ -679,20 +685,39 @@ proc constructObject*[O: object|tuple]( inc(i) else: ensureAllFieldsPresent(s, O, result, matched) +macro genRepresentObject(t: typedesc, value, childTagStyle: typed): typed = + echo "here" + result = quote do: + when compiles(`transientBitvectorProc`(`t`)): + const bitvector = `transientBitvectorProc`(`t`) + var fieldIndex = 0'i16 + for name, value in fieldPairs(value): + when compiles(`transientBitvectorProc`(`t`)): + if fieldIndex notin bitvector: + when isVariantObject(O): c.put(startMapEvent(yTagQuestionMark, yAnchorNone)) + c.put(scalarEvent(name, if childTagStyle == tsNone: yTagQuestionMark else: + yTagNimField, yAnchorNone)) + representChild(value, childTagStyle, c) + when isVariantObject(O): c.put(endMapEvent()) + fieldIndex.inc() + else: + when isVariantObject(O): c.put(startMapEvent(yTagQuestionMark, yAnchorNone)) + c.put(scalarEvent(name, if childTagStyle == tsNone: yTagQuestionMark else: + yTagNimField, yAnchorNone)) + representChild(value, childTagStyle, c) + when isVariantObject(O): c.put(endMapEvent()) + echo "did it" + proc representObject*[O: object|tuple](value: O, ts: TagStyle, c: SerializationContext, tag: TagId) = ## represents a Nim object or tuple as YAML mapping let childTagStyle = if ts == tsRootOnly: tsNone else: ts when isVariantObject(O): c.put(startSeqEvent(tag, yAnchorNone)) else: c.put(startMapEvent(tag, yAnchorNone)) - for name, value in fieldPairs(value): - when isVariantObject(O): c.put(startMapEvent(yTagQuestionMark, yAnchorNone)) - c.put(scalarEvent(name, if childTagStyle == tsNone: yTagQuestionMark else: - yTagNimField, yAnchorNone)) - representChild(value, childTagStyle, c) - when isVariantObject(O): c.put(endMapEvent()) + genRepresentObject(O, value, childTagStyle) when isVariantObject(O): c.put(endSeqEvent()) else: c.put(endMapEvent()) + static: echo "represented " & typetraits.name(O) proc constructObject*[O: enum](s: var YamlStream, c: ConstructionContext, result: var O) @@ -754,9 +779,6 @@ macro constructImplicitVariantObject(s, c, r, possibleTagIds: untyped, )))) result = newStmtList(newCall("reset", r), ifStmt) -let implicitVariantObjectMarker {.compileTime.} = - newIdentNode(":implicitVariantObject") - macro isImplicitVariantObject(o: typed): untyped = result = newCall("compiles", newCall(implicitVariantObjectMarker, o)) @@ -941,8 +963,10 @@ proc representChild*[O](value: O, ts: TagStyle, inc(count) if count == 1: c.put(scalarEvent("~", yTagNull)) else: + static: echo "repchild " & typetraits.name(O) representObject(value, ts, c, if ts == tsNone: yTagQuestionMark else: yamlTag(O)) + static: echo "/repchild " & typetraits.name(O) proc construct*[T](s: var YamlStream, target: var T) {.raises: [YamlStreamError].} = @@ -1067,6 +1091,8 @@ proc canBeImplicit(t: typedesc): bool {.compileTime.} = macro setImplicitVariantObjectMarker(t: typedesc): untyped = result = quote do: + when compiles(`transientBitvectorProc`(`t`)): + {.fatal: "Cannot mark object with transient fields as implicit".} proc `implicitVariantObjectMarker`*(unused: `t`) = discard template markAsImplicit*(t: typedesc): typed = @@ -1080,3 +1106,46 @@ template markAsImplicit*(t: typedesc): typed = else: {. fatal: "This type cannot be marked as implicit" .} +macro getFieldIndex(t: typedesc, field: untyped): untyped = + let + tDecl = getType(t) + tName = $tDecl[1] + tDesc = getType(tDecl[1]) + fieldName = $field + var + fieldIndex = 0 + found = false + block outer: + for child in tDesc[2].children: + if child.kind == nnkRecCase: + for bIndex in 1 .. len(child) - 1: + for item in child[bIndex][1].children: + inc(fieldIndex) + yAssert item.kind == nnkSym + if $item == fieldName: + found = true + break outer + else: + yAssert child.kind == nnkSym + if $child == fieldName: + found = true + break + inc(fieldIndex) + if not found: + let msg = "Type " & tName & " has no field " & fieldName & '!' + result = quote do: {.fatal: `msg`.} + else: + result = newNimNode(nnkInt16Lit) + result.intVal = fieldIndex + +macro markAsTransient*(t: typedesc, field: untyped): typed = + let mySet = genSym(nskVar, ident = ":mySym") + quote do: + when compiles(`implicitVariantObjectMarker`(`t`)): + {.fatal: "Cannot mark fields of implicit variant objects as transient." .} + when not compiles(`transientBitvectorProc`(`t`)): + var `mySet` {.compileTime.}: set[int16] = {} + proc `transientBitvectorProc`*(myType: typedesc[`t`]): var set[int16] + {.compileTime.} = + return `mySet` + static: `transientBitvectorProc`(`t`).incl(getFieldIndex(`t`, `field`)) \ No newline at end of file From 3ab3dc7ad0575805e632bec7a7a4c6af6a6e1d66 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sat, 15 Oct 2016 17:58:29 +0200 Subject: [PATCH 16/26] Dump variant objects with transient fields * tuples not working yet * loading not woring yet * added tests --- test/tserialization.nim | 51 +++++++++++++ yaml/serialization.nim | 156 +++++++++++++++++++++++++++++++--------- 2 files changed, 174 insertions(+), 33 deletions(-) diff --git a/test/tserialization.nim b/test/tserialization.nim index 4432626..51895f5 100644 --- a/test/tserialization.nim +++ b/test/tserialization.nim @@ -38,6 +38,31 @@ type of akDog: barkometer: int + DumbEnum = enum + deA, deB, deC, deD + + NonVariantWithTransient = object + a, b, c, d: string + + VariantWithTransient = object + gStorable, gTemporary: string + case kind: DumbEnum + of deA, deB: + cStorable, cTemporary: string + of deC: + alwaysThere: int + of deD: + neverThere: int + +markAsTransient(NonVariantWithTransient, a) +markAsTransient(NonVariantWithTransient, c) + +markAsTransient(VariantWithTransient, gTemporary) +markAsTransient(VariantWithTransient, cTemporary) +markAsTransient(VariantWithTransient, neverThere) + + + proc `$`(v: BetterInt): string {.borrow.} proc `==`(left, right: BetterInt): bool {.borrow.} @@ -451,6 +476,32 @@ suite "Serialization": expectConstructionError(1, 32, "While constructing Animal: Missing field: \"purringIntensity\""): load(input, result) + test "Dump non-variant object with transient fields": + let input = NonVariantWithTransient(a: "a", b: "b", c: "c", d: "d") + let output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual yamlDirs & "\nb: b\nd: d", output + + test "Dump variant object with transient fields": + let input = @[VariantWithTransient(kind: deB, gStorable: "gs", + gTemporary: "gt", cStorable: "cs", cTemporary: "ct"), + VariantWithTransient(kind: deD, gStorable: "a", gTemporary: "b", + neverThere: 42)] + let output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual yamlDirs & """ + +- + - + gStorable: gs + - + kind: deB + - + cStorable: cs +- + - + gStorable: a + - + kind: deD""", output + test "Dump cyclic data structure": var a = newNode("a") diff --git a/yaml/serialization.nim b/yaml/serialization.nim index 5debffb..f909b18 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -616,8 +616,10 @@ macro constructFieldValue(t: typedesc, stream: untyped, context: untyped, newCall(bindSym("escape"), name)))))) proc isVariantObject(t: typedesc): bool {.compileTime.} = - let tDesc = getType(t) - if tDesc.kind != nnkObjectTy: return false + var tDesc = getType(t) + if tDesc.kind == nnkBracketExpr: tDesc = getType(tDesc[1]) + if tDesc.kind != nnkObjectTy: + return false for child in tDesc[2].children: if child.kind == nnkRecCase: return true return false @@ -628,6 +630,8 @@ let transientBitvectorProc {.compileTime.} = newIdentNode(":transientObject") +var transientVectors {.compileTime.} = newSeq[set[int16]]() + proc constructObject*[O: object|tuple]( s: var YamlStream, c: ConstructionContext, result: var O) {.raises: [YamlConstructionError, YamlStreamError].} = @@ -685,30 +689,92 @@ proc constructObject*[O: object|tuple]( inc(i) else: ensureAllFieldsPresent(s, O, result, matched) -macro genRepresentObject(t: typedesc, value, childTagStyle: typed): typed = - echo "here" +macro getTransientBitvector(t: typedesc): untyped = + echo "getTransientBitvector" result = quote do: when compiles(`transientBitvectorProc`(`t`)): - const bitvector = `transientBitvectorProc`(`t`) - var fieldIndex = 0'i16 - for name, value in fieldPairs(value): - when compiles(`transientBitvectorProc`(`t`)): - if fieldIndex notin bitvector: - when isVariantObject(O): c.put(startMapEvent(yTagQuestionMark, yAnchorNone)) - c.put(scalarEvent(name, if childTagStyle == tsNone: yTagQuestionMark else: - yTagNimField, yAnchorNone)) - representChild(value, childTagStyle, c) - when isVariantObject(O): c.put(endMapEvent()) - fieldIndex.inc() - else: - when isVariantObject(O): c.put(startMapEvent(yTagQuestionMark, yAnchorNone)) - c.put(scalarEvent(name, if childTagStyle == tsNone: yTagQuestionMark else: - yTagNimField, yAnchorNone)) - representChild(value, childTagStyle, c) - when isVariantObject(O): c.put(endMapEvent()) - echo "did it" + transientVectors[`transientBitvectorProc`(`t`)] + else: + set[int16]({}) + echo result.repr -proc representObject*[O: object|tuple](value: O, ts: TagStyle, +macro genRepresentObject(t: typedesc, value, childTagStyle: typed): typed = + result = newStmtList() + let tSym = genSym(nskConst, ":tSym") + result.add(quote do: + when compiles(`transientBitvectorProc`(`t`)): + const `tSym` = `transientBitvectorProc`(`t`) + else: + const `tSym` = -1 + ) + let + tDecl = getType(t) + tDesc = getType(tDecl[1]) + isVO = isVariantObject(t) + var fieldIndex = 0'i16 + for child in tDesc[2].children: + if child.kind == nnkRecCase: + let + fieldName = $child[0] + fieldAccessor = newDotExpr(value, newIdentNode(fieldName)) + result.add(quote do: + c.put(startMapEvent(yTagQuestionMark, yAnchorNone)) + c.put(scalarEvent(`fieldName`, if `childTagStyle` == tsNone: + yTagQuestionMark else: yTagNimField, yAnchorNone)) + representChild(`fieldAccessor`, `childTagStyle`, c) + c.put(endMapEvent()) + ) + let enumName = $getTypeInst(child[0]) + var caseStmt = newNimNode(nnkCaseStmt).add(fieldAccessor) + for bIndex in 1 .. len(child) - 1: + var curBranch: NimNode + var recListIndex = 0 + case child[bIndex].kind + of nnkOfBranch: + curBranch = newNimNode(nnkOfBranch) + while child[bIndex][recListIndex].kind == nnkIntLit: + curBranch.add(newCall(enumName, newLit(child[bIndex][recListIndex].intVal))) + inc(recListIndex) + of nnkElse: + curBranch = newNimNode(nnkElse) + else: + internalError("Unexpected child kind: " & $child[bIndex].kind) + doAssert child[bIndex][recListIndex].kind == nnkRecList + var curStmtList = newStmtList() + if child[bIndex][recListIndex].len > 0: + for item in child[bIndex][recListIndex].children: + inc(fieldIndex) + let + name = $item + itemAccessor = newDotExpr(value, newIdentNode(name)) + curStmtList.add(quote do: + when `tSym` == -1 or `fieldIndex` notin transientVectors[`tSym`]: + c.put(startMapEvent(yTagQuestionMark, yAnchorNone)) + c.put(scalarEvent(`name`, if `childTagStyle` == tsNone: + yTagQuestionMark else: yTagNimField, yAnchorNone)) + representChild(`itemAccessor`, `childTagStyle`, c) + c.put(endMapEvent()) + ) + else: + curStmtList.add(newNimNode(nnkDiscardStmt).add(newEmptyNode())) + curBranch.add(curStmtList) + caseStmt.add(curBranch) + result.add(caseStmt) + else: + let + name = $child + childAccessor = newDotExpr(value, newIdentNode(name)) + result.add(quote do: + when `tSym` == -1 or `fieldIndex` notin transientVectors[`tSym`]: + when `isVO`: c.put(startMapEvent(yTagQuestionMark, yAnchorNone)) + c.put(scalarEvent(`name`, if `childTagStyle` == tsNone: + yTagQuestionMark else: yTagNimField, yAnchorNone)) + representChild(`childAccessor`, `childTagStyle`, c) + when `isVO`: c.put(endMapEvent()) + ) + inc(fieldIndex) + +proc representObject*[O: object](value: O, ts: TagStyle, c: SerializationContext, tag: TagId) = ## represents a Nim object or tuple as YAML mapping let childTagStyle = if ts == tsRootOnly: tsNone else: ts @@ -717,7 +783,20 @@ proc representObject*[O: object|tuple](value: O, ts: TagStyle, genRepresentObject(O, value, childTagStyle) when isVariantObject(O): c.put(endSeqEvent()) else: c.put(endMapEvent()) - static: echo "represented " & typetraits.name(O) + +proc representObject*[O: tuple](value: O, ts: TagStyle, + c: SerializationContext, tag: TagId) = + let childTagStyle = if ts == tsRootOnly: tsNone else: ts + const bitvector = getTransientBitvector(O) + var fieldIndex = 0'i16 + c.put(startMapEvent(tag, yAnchorNone)) + for name, fvalue in fieldPairs(value): + #if fieldIndex notin bitvector: TODO: fix! + c.put(scalarEvent(name, if childTagStyle == tsNone: + yTagQuestionMark else: yTagNimField, yAnchorNone)) + representChild(fvalue, childTagStyle, c) + inc(fieldIndex) + c.put(endMapEvent()) proc constructObject*[O: enum](s: var YamlStream, c: ConstructionContext, result: var O) @@ -963,10 +1042,8 @@ proc representChild*[O](value: O, ts: TagStyle, inc(count) if count == 1: c.put(scalarEvent("~", yTagNull)) else: - static: echo "repchild " & typetraits.name(O) representObject(value, ts, c, if ts == tsNone: yTagQuestionMark else: yamlTag(O)) - static: echo "/repchild " & typetraits.name(O) proc construct*[T](s: var YamlStream, target: var T) {.raises: [YamlStreamError].} = @@ -1119,7 +1196,16 @@ macro getFieldIndex(t: typedesc, field: untyped): untyped = for child in tDesc[2].children: if child.kind == nnkRecCase: for bIndex in 1 .. len(child) - 1: - for item in child[bIndex][1].children: + var bChildIndex = 0 + case child[bIndex].kind + of nnkOfBranch: + while child[bIndex][bChildIndex].kind == nnkIntLit: inc(bChildIndex) + of nnkElse: discard + else: + internalError("Unexpected child kind: " & + $child[bIndex][bChildIndex].kind) + yAssert child[bIndex][bChildIndex].kind == nnkRecList + for item in child[bIndex][bChildIndex].children: inc(fieldIndex) yAssert item.kind == nnkSym if $item == fieldName: @@ -1137,15 +1223,19 @@ macro getFieldIndex(t: typedesc, field: untyped): untyped = else: result = newNimNode(nnkInt16Lit) result.intVal = fieldIndex + echo "found fieldIndex of \"", fieldName, "\": ", result.intVal macro markAsTransient*(t: typedesc, field: untyped): typed = - let mySet = genSym(nskVar, ident = ":mySym") - quote do: + let nextBitvectorIndex = transientVectors.len + result = quote do: when compiles(`implicitVariantObjectMarker`(`t`)): {.fatal: "Cannot mark fields of implicit variant objects as transient." .} when not compiles(`transientBitvectorProc`(`t`)): - var `mySet` {.compileTime.}: set[int16] = {} - proc `transientBitvectorProc`*(myType: typedesc[`t`]): var set[int16] + proc `transientBitvectorProc`*(myType: typedesc[`t`]): int {.compileTime.} = - return `mySet` - static: `transientBitvectorProc`(`t`).incl(getFieldIndex(`t`, `field`)) \ No newline at end of file + `nextBitvectorIndex` + static: transientVectors.add({}) + static: + echo "setting transientBitvector at ", `transientBitvectorProc`(`t`) + transientVectors[`transientBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`)) + echo "transient bitvector is now ", transientVectors[`transientBitvectorProc`(`t`)] \ No newline at end of file From ab8bfe419219c2499f2c09ac3a5f8d6da997e4e0 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Sat, 15 Oct 2016 18:30:52 +0200 Subject: [PATCH 17/26] Removed garbage code --- yaml/serialization.nim | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/yaml/serialization.nim b/yaml/serialization.nim index f909b18..bbf3e5b 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -689,15 +689,6 @@ proc constructObject*[O: object|tuple]( inc(i) else: ensureAllFieldsPresent(s, O, result, matched) -macro getTransientBitvector(t: typedesc): untyped = - echo "getTransientBitvector" - result = quote do: - when compiles(`transientBitvectorProc`(`t`)): - transientVectors[`transientBitvectorProc`(`t`)] - else: - set[int16]({}) - echo result.repr - macro genRepresentObject(t: typedesc, value, childTagStyle: typed): typed = result = newStmtList() let tSym = genSym(nskConst, ":tSym") @@ -787,11 +778,9 @@ proc representObject*[O: object](value: O, ts: TagStyle, proc representObject*[O: tuple](value: O, ts: TagStyle, c: SerializationContext, tag: TagId) = let childTagStyle = if ts == tsRootOnly: tsNone else: ts - const bitvector = getTransientBitvector(O) var fieldIndex = 0'i16 c.put(startMapEvent(tag, yAnchorNone)) for name, fvalue in fieldPairs(value): - #if fieldIndex notin bitvector: TODO: fix! c.put(scalarEvent(name, if childTagStyle == tsNone: yTagQuestionMark else: yTagNimField, yAnchorNone)) representChild(fvalue, childTagStyle, c) @@ -1223,7 +1212,6 @@ macro getFieldIndex(t: typedesc, field: untyped): untyped = else: result = newNimNode(nnkInt16Lit) result.intVal = fieldIndex - echo "found fieldIndex of \"", fieldName, "\": ", result.intVal macro markAsTransient*(t: typedesc, field: untyped): typed = let nextBitvectorIndex = transientVectors.len @@ -1236,6 +1224,4 @@ macro markAsTransient*(t: typedesc, field: untyped): typed = `nextBitvectorIndex` static: transientVectors.add({}) static: - echo "setting transientBitvector at ", `transientBitvectorProc`(`t`) transientVectors[`transientBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`)) - echo "transient bitvector is now ", transientVectors[`transientBitvectorProc`(`t`)] \ No newline at end of file From 9290ca0d91561aba3c81b7ce1a3010f360f6cdc1 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Wed, 19 Oct 2016 22:04:46 +0200 Subject: [PATCH 18/26] Load objects with transient fields --- test/tserialization.nim | 36 +++++++++++- yaml/serialization.nim | 123 ++++++++++++++++++++++++++++++---------- 2 files changed, 127 insertions(+), 32 deletions(-) diff --git a/test/tserialization.nim b/test/tserialization.nim index 51895f5..b049ae4 100644 --- a/test/tserialization.nim +++ b/test/tserialization.nim @@ -105,8 +105,8 @@ template expectConstructionError(li, co: int, message: string, body: typed) = let e = (ref YamlConstructionError)(getCurrentException()) doAssert li == e.line, "Expected error line " & $li & ", was " & $e.line doAssert co == e.column, "Expected error column " & $co & ", was " & $e.column - doAssert message == e.msg, "Expected error message " & escape(message) & - ", got " & escape(e.msg) + doAssert message == e.msg, "Expected error message \n" & escape(message) & + ", got \n" & escape(e.msg) proc newNode(v: string): ref Node = new(result) @@ -476,11 +476,43 @@ suite "Serialization": expectConstructionError(1, 32, "While constructing Animal: Missing field: \"purringIntensity\""): load(input, result) + test "Load non-variant object with transient fields": + let input = "{b: b, d: d}" + var result: NonVariantWithTransient + load(input, result) + assert isNil(result.a) + assert result.b == "b" + assert isNil(result.c) + assert result.d == "d" + + test "Load non-variant object with transient fields - unknown field": + let input = "{b: b, c: c, d: d}" + var result: NonVariantWithTransient + expectConstructionError(1, 9, "While constructing NonVariantWithTransient: Field \"c\" is transient and may not occur in input"): + load(input, result) + test "Dump non-variant object with transient fields": let input = NonVariantWithTransient(a: "a", b: "b", c: "c", d: "d") let output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual yamlDirs & "\nb: b\nd: d", output + test "Load variant object with transient fields": + let input = "[[gStorable: gs, kind: deB, cStorable: cs], [gStorable: a, kind: deD]]" + var result: seq[VariantWithTransient] + load(input, result) + assert result.len == 2 + assert result[0].kind == deB + assert result[0].gStorable == "gs" + assert result[0].cStorable == "cs" + assert result[1].kind == deD + assert result[1].gStorable == "a" + + test "Load variant object with transient fields": + let input = "[gStorable: gc, kind: deD, neverThere: foo]" + var result: VariantWithTransient + expectConstructionError(1, 38, "While constructing VariantWithTransient: Field \"neverThere\" is transient and may not occur in input"): + load(input, result) + test "Dump variant object with transient fields": let input = @[VariantWithTransient(kind: deB, gStorable: "gs", gTemporary: "gt", cStorable: "cs", cTemporary: "ct"), diff --git a/yaml/serialization.nim b/yaml/serialization.nim index bbf3e5b..89f72dd 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -513,7 +513,15 @@ proc fieldCount(t: typedesc): int {.compileTime.} = inc(result) if child.kind == nnkRecCase: for bIndex in 1.. recListIndex: + inc(result, child[bIndex][recListIndex].len) macro matchMatrix(t: typedesc): untyped = result = newNimNode(nnkBracket) @@ -539,7 +547,28 @@ proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} = newAssignment(newNimNode(nnkBracketExpr).add(matched, newLit(i)), newLit(true)) -macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed, +let + implicitVariantObjectMarker {.compileTime.} = + newIdentNode(":implicitVariantObject") + transientBitvectorProc {.compileTime.} = + newIdentNode(":transientObject") + +var transientVectors {.compileTime.} = newSeq[set[int16]]() + +proc ifNotTransient(tSym: NimNode, fieldIndex: int, content: openarray[NimNode], + elseError: bool = false, s: NimNode = nil, tName, fName: string = nil): + NimNode {.compileTime.} = + var stmts = newStmtList(content) + result = quote do: + when `tSym` == -1 or `fieldIndex` notin transientVectors[`tSym`]: + `stmts` + if elseError: + result[0].add(newNimNode(nnkElse).add(quote do: + raise constructionError(`s`, "While constructing " & `tName` & + ": Field \"" & `fName` & "\" is transient and may not occur in input") + )) + +macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, tIndex: int, o: typed, matched: typed): typed = result = newStmtList() let @@ -552,22 +581,44 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, o: typed, result.add(checkMissing(s, tName, $child[0], field, matched)) for bIndex in 1 .. len(child) - 1: let discChecks = newStmtList() - for item in child[bIndex][1].children: + var + curValues = newNimNode(nnkCurly) + recListIndex = 0 + case child[bIndex].kind + of nnkOfBranch: + while recListIndex < child[bIndex].len and + child[bIndex][recListIndex].kind == nnkIntLit: + curValues.add(child[bIndex][recListIndex]) + inc(recListIndex) + of nnkElse: discard + else: internalError("Unexpected child kind: " & $child[bIndex].kind) + for item in child[bIndex][recListIndex].children: inc(field) discChecks.add(checkMissing(s, tName, $item, field, matched)) - result.add(newIfStmt((infix(newDotExpr(o, newIdentNode($child[0])), - "==", child[bIndex][0]), discChecks))) + result.add(ifNotTransient(tIndex, field, + [newIfStmt((infix(newDotExpr(o, newIdentNode($child[0])), + "in", curValues), discChecks))])) else: - result.add(checkMissing(s, tName, $child, field, matched)) + result.add(ifNotTransient(tIndex, field, + [checkMissing(s, tName, $child, field, matched)])) inc(field) -macro constructFieldValue(t: typedesc, stream: untyped, context: untyped, - name: untyped, o: untyped, matched: untyped): typed = +macro fetchTransientIndex(t: typedesc, tIndex: untyped): typed = + quote do: + when compiles(`transientBitvectorProc`(`t`)): + const `tIndex` = `transientBitvectorProc`(`t`) + else: + const `tIndex` = -1 + +macro constructFieldValue(t: typedesc, tIndex: int, stream: untyped, + context: untyped, name: untyped, o: untyped, + matched: untyped): typed = let tDecl = getType(t) tName = $tDecl[1] tDesc = getType(tDecl[1]) - result = newNimNode(nnkCaseStmt).add(name) + result = newStmtList() + var caseStmt = newNimNode(nnkCaseStmt).add(name) var fieldIndex = 0 for child in tDesc[2].children: if child.kind == nnkRecCase: @@ -583,10 +634,26 @@ macro constructFieldValue(t: typedesc, stream: untyped, context: untyped, newCall("constructChild", stream, context, newIdentNode("value")), newAssignment(discriminant, newIdentNode("value")), markAsFound(fieldIndex, matched))) - result.add(disOb) + caseStmt.add(disOb) + var alreadyUsedSet = newNimNode(nnkCurly) for bIndex in 1 .. len(child) - 1: - let discTest = infix(discriminant, "==", child[bIndex][0]) - for item in child[bIndex][1].children: + var recListIndex = 0 + var discTest: NimNode + case child[bIndex].kind + of nnkOfBranch: + discTest = newNimNode(nnkCurly) + while child[bIndex][recListIndex].kind == nnkIntLit: + discTest.add(child[bIndex][recListIndex]) + alreadyUsedSet.add(child[bIndex][recListIndex]) + inc(recListIndex) + discTest = infix(discriminant, "in", discTest) + of nnkElse: + discTest = infix(discriminant, "notin", alreadyUsedSet) + else: + internalError("Unexpected child kind: " & $child[bIndex].kind) + doAssert child[bIndex][recListIndex].kind == nnkRecList + + for item in child[bIndex][recListIndex].children: inc(fieldIndex) yAssert item.kind == nnkSym var ob = newNimNode(nnkOfBranch).add(newStrLitNode($item)) @@ -597,23 +664,26 @@ macro constructFieldValue(t: typedesc, stream: untyped, context: untyped, newCall(bindSym("constructionError"), stream, infix(newStrLitNode("Field " & $item & " not allowed for " & $child[0] & " == "), "&", prefix(discriminant, "$")))))) - ob.add(newStmtList(checkDuplicate(stream, tName, $item, fieldIndex, - matched), ifStmt, markAsFound(fieldIndex, matched))) - result.add(ob) + ob.add(ifNotTransient(tIndex, fieldIndex, + [checkDuplicate(stream, tName, $item, fieldIndex, matched), + ifStmt, markAsFound(fieldIndex, matched)], true, stream, tName, + $item)) + caseStmt.add(ob) else: yAssert child.kind == nnkSym var ob = newNimNode(nnkOfBranch).add(newStrLitNode($child)) let field = newDotExpr(o, newIdentNode($child)) - ob.add(newStmtList( - checkDuplicate(stream, tName, $child, fieldIndex, matched), + ob.add(ifNotTransient(tIndex, fieldIndex, + [checkDuplicate(stream, tName, $child, fieldIndex, matched), newCall("constructChild", stream, context, field), - markAsFound(fieldIndex, matched))) - result.add(ob) + markAsFound(fieldIndex, matched)], true, stream, tName, $child)) + caseStmt.add(ob) inc(fieldIndex) - result.add(newNimNode(nnkElse).add(newNimNode(nnkRaiseStmt).add( + caseStmt.add(newNimNode(nnkElse).add(newNimNode(nnkRaiseStmt).add( newCall(bindSym("constructionError"), stream, infix(newLit("While constructing " & tName & ": Unknown field: "), "&", newCall(bindSym("escape"), name)))))) + result.add(caseStmt) proc isVariantObject(t: typedesc): bool {.compileTime.} = var tDesc = getType(t) @@ -624,14 +694,6 @@ proc isVariantObject(t: typedesc): bool {.compileTime.} = if child.kind == nnkRecCase: return true return false -let - implicitVariantObjectMarker {.compileTime.} = - newIdentNode(":implicitVariantObject") - transientBitvectorProc {.compileTime.} = - newIdentNode(":transientObject") - -var transientVectors {.compileTime.} = newSeq[set[int16]]() - proc constructObject*[O: object|tuple]( s: var YamlStream, c: ConstructionContext, result: var O) {.raises: [YamlConstructionError, YamlStreamError].} = @@ -641,6 +703,7 @@ proc constructObject*[O: object|tuple]( const startKind = when isVariantObject(O): yamlStartSeq else: yamlStartMap endKind = when isVariantObject(O): yamlEndSeq else: yamlEndMap + fetchTransientIndex(O, tIndex) if e.kind != startKind: raise s.constructionError("While constructing " & typetraits.name(O) & ": Expected " & $startKind & ", got " & $e.kind) @@ -673,7 +736,7 @@ proc constructObject*[O: object|tuple]( raise s.constructionError("While constructing " & typetraits.name(O) & ": Unknown field: " & escape(name)) else: - constructFieldValue(O, s, c, name, result, matched) + constructFieldValue(O, tIndex, s, c, name, result, matched) when isVariantObject(O): e = s.next() if e.kind != yamlEndMap: @@ -687,7 +750,7 @@ proc constructObject*[O: object|tuple]( raise s.constructionError("While constructing " & typetraits.name(O) & ": Missing field: " & escape(fname)) inc(i) - else: ensureAllFieldsPresent(s, O, result, matched) + else: ensureAllFieldsPresent(s, O, tIndex, result, matched) macro genRepresentObject(t: typedesc, value, childTagStyle: typed): typed = result = newStmtList() From fd05f828ad450126f50241f7edd94ef1313db3fa Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Thu, 20 Oct 2016 18:12:35 +0200 Subject: [PATCH 19/26] Updated serialization doc --- doc/serialization.txt | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/doc/serialization.txt b/doc/serialization.txt index 8835a55..f19ba54 100644 --- a/doc/serialization.txt +++ b/doc/serialization.txt @@ -247,6 +247,33 @@ for example use it on a certain ``seq`` type: setTagUri(seq[string], "!nim:my:seq") +Customizing Field Handling +========================== + +NimYAML allows the user to specify special handling of certain object fields. +This configuration will be applied at compile time when NimYAML generates the +(de)serialization code for an object type. It is important that the +configuration happens before any YAML operations (e.g. ``load`` or ``dump``) are +executed on a variable of the object type. + +Transient Fields +---------------- + +It may happen that certain fields of an object type are transient, i.e. they are +used in a way that makes (de)serializing them unnecessary. Such fields can be +marked as transient. This will cause them not to be serialized to YAML. They +will also not be accepted when loading the object. + +Example: + +.. code-block:: nim + + type MyObject: object + storable: string + temporary: string + + markAsTransient(MyObject, temporary) + Customize Serialization ======================= From 6402433d2a62865ab6b929bd629fac34a8a833d0 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Wed, 26 Oct 2016 18:32:54 +0200 Subject: [PATCH 20/26] Implemented setDefaultValue. --- test/tserialization.nim | 24 +++++++++++- yaml/serialization.nim | 81 +++++++++++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/test/tserialization.nim b/test/tserialization.nim index b049ae4..f701f12 100644 --- a/test/tserialization.nim +++ b/test/tserialization.nim @@ -54,6 +54,9 @@ type of deD: neverThere: int + WithDefault = object + a, b, c, d: string + markAsTransient(NonVariantWithTransient, a) markAsTransient(NonVariantWithTransient, c) @@ -61,7 +64,8 @@ markAsTransient(VariantWithTransient, gTemporary) markAsTransient(VariantWithTransient, cTemporary) markAsTransient(VariantWithTransient, neverThere) - +setDefaultValue(WithDefault, b, "b") +setDefaultValue(WithDefault, d, "d") proc `$`(v: BetterInt): string {.borrow.} proc `==`(left, right: BetterInt): bool {.borrow.} @@ -579,6 +583,24 @@ next: assert(result[1].next == result[2]) assert(result[2].next == result[0]) + test "Load object with default values": + let input = "a: abc\nc: dce" + var result: WithDefault + load(input, result) + assert result.a == "abc" + assert result.b == "b" + assert result.c == "dce" + assert result.d == "d" + + test "Load object with partly default values": + let input = "a: abc\nb: bcd\nc: cde" + var result: WithDefault + load(input, result) + assert result.a == "abc" + assert result.b == "bcd" + assert result.c == "cde" + assert result.d == "d" + test "Load nil values": let input = newStringStream("- ~\n- !!str ~") var result: seq[ref string] diff --git a/yaml/serialization.nim b/yaml/serialization.nim index 89f72dd..99c559a 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -536,24 +536,45 @@ proc checkDuplicate(s: NimNode, tName: string, name: string, i: int, newLit("While constructing " & tName & ": Duplicate field: " & escape(name)))))) -proc checkMissing(s: NimNode, tName: string, name: string, i: int, - matched: NimNode): NimNode {.compileTime.} = - result = newIfStmt((newCall("not", newNimNode(nnkBracketExpr).add(matched, - newLit(i))), newNimNode(nnkRaiseStmt).add(newCall( - bindSym("constructionError"), s, newLit("While constructing " & - tName & ": Missing field: " & escape(name)))))) - -proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} = - newAssignment(newNimNode(nnkBracketExpr).add(matched, newLit(i)), - newLit(true)) - let implicitVariantObjectMarker {.compileTime.} = newIdentNode(":implicitVariantObject") transientBitvectorProc {.compileTime.} = newIdentNode(":transientObject") + defaultBitvectorProc {.compileTime.} = + newIdentNode(":defaultBitvector") + defaultValueGetter {.compileTime.} = + newIdentNode(":defaultValueGetter") -var transientVectors {.compileTime.} = newSeq[set[int16]]() +var + transientVectors {.compileTime.} = newSeq[set[int16]]() + defaultVectors {.compileTime.} = newSeq[set[int16]]() + +proc addDefaultOr(tName: string, i: int, o: NimNode, + field, elseBranch, defaultValues: NimNode): NimNode {.compileTime.} = + let + dbp = defaultBitvectorProc + t = newIdentNode(tName) + result = quote do: + when compiles(`dbp`(`t`)): + when `i` in defaultVectors[`dbp`(`t`)]: + `o`.`field` = `defaultValues`.`field` + else: `elseBranch` + else: `elseBranch` + +proc checkMissing(s: NimNode, t: typedesc, tName: string, field: NimNode, + i: int, matched, o, defaultValues: NimNode): + NimNode {.compileTime.} = + result = newIfStmt((newCall("not", newNimNode(nnkBracketExpr).add(matched, + newLit(i))), addDefaultOr(tName, i, o, field, + newNimNode(nnkRaiseStmt).add(newCall( + bindSym("constructionError"), s, newLit("While constructing " & + tName & ": Missing field: " & escape($field)))), defaultValues))) + echo result.repr + +proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} = + newAssignment(newNimNode(nnkBracketExpr).add(matched, newLit(i)), + newLit(true)) proc ifNotTransient(tSym: NimNode, fieldIndex: int, content: openarray[NimNode], elseError: bool = false, s: NimNode = nil, tName, fName: string = nil): @@ -570,7 +591,12 @@ proc ifNotTransient(tSym: NimNode, fieldIndex: int, content: openarray[NimNode], macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, tIndex: int, o: typed, matched: typed): typed = - result = newStmtList() + let + dbp = defaultBitvectorProc + defaultValues = genSym(nskConst, "defaultValues") + result = quote do: + when compiles(`dbp`(`t`)): + const `defaultValues` = `defaultValueGetter`(`t`) let tDecl = getType(t) tName = $tDecl[1] @@ -578,7 +604,8 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, tIndex: int, o: typed, var field = 0 for child in tDesc[2].children: if child.kind == nnkRecCase: - result.add(checkMissing(s, tName, $child[0], field, matched)) + result.add(checkMissing(s, t, tName, child[0], field, matched, o, + defaultValues)) for bIndex in 1 .. len(child) - 1: let discChecks = newStmtList() var @@ -594,13 +621,14 @@ macro ensureAllFieldsPresent(s: YamlStream, t: typedesc, tIndex: int, o: typed, else: internalError("Unexpected child kind: " & $child[bIndex].kind) for item in child[bIndex][recListIndex].children: inc(field) - discChecks.add(checkMissing(s, tName, $item, field, matched)) + discChecks.add(checkMissing(s, t, tName, item, field, matched, o, + defaultValues)) result.add(ifNotTransient(tIndex, field, [newIfStmt((infix(newDotExpr(o, newIdentNode($child[0])), "in", curValues), discChecks))])) else: result.add(ifNotTransient(tIndex, field, - [checkMissing(s, tName, $child, field, matched)])) + [checkMissing(s, t, tName, child, field, matched, o, defaultValues)])) inc(field) macro fetchTransientIndex(t: typedesc, tIndex: untyped): typed = @@ -1288,3 +1316,24 @@ macro markAsTransient*(t: typedesc, field: untyped): typed = static: transientVectors.add({}) static: transientVectors[`transientBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`)) + +macro setDefaultValue*(t: typedesc, field: untyped, value: typed): typed = + let + dSym = genSym(nskVar, ":default") + nextBitvectorIndex = defaultVectors.len + result = quote do: + when compiles(`transientBitvectorProc`(`t`)): + {.fatal: "Cannot set default value of transient field".} + elif compiles(`implicitVariantObjectMarker`(`t`)): + {.fatal: "Cannot set default value of implicit variant objects.".} + when not compiles(`defaultValueGetter`(`t`)): + var `dSym` {.compileTime.}: `t` + template `defaultValueGetter`(t: typedesc[`t`]): auto = + `dSym` + proc `defaultBitvectorProc`*(myType: typedesc[`t`]): int + {.compileTime.} = `nextBitvectorIndex` + static: defaultVectors.add({}) + static: + `defaultValueGetter`(`t`).`field` = `value` + defaultVectors[`defaultBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`)) + echo result.repr \ No newline at end of file From dcfa8fd27cab39d33ad9690e04d8359c60592ce2 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Thu, 27 Oct 2016 17:58:14 +0200 Subject: [PATCH 21/26] Implemented ignoreInputKey --- test/tserialization.nim | 23 +++++++++++++++++++++++ yaml/serialization.nim | 39 ++++++++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/test/tserialization.nim b/test/tserialization.nim index f701f12..c089926 100644 --- a/test/tserialization.nim +++ b/test/tserialization.nim @@ -57,6 +57,9 @@ type WithDefault = object a, b, c, d: string + WithIgnoredField = object + x, y: int + markAsTransient(NonVariantWithTransient, a) markAsTransient(NonVariantWithTransient, c) @@ -67,6 +70,8 @@ markAsTransient(VariantWithTransient, neverThere) setDefaultValue(WithDefault, b, "b") setDefaultValue(WithDefault, d, "d") +ignoreInputKey(WithIgnoredField, "z") + proc `$`(v: BetterInt): string {.borrow.} proc `==`(left, right: BetterInt): bool {.borrow.} @@ -538,6 +543,24 @@ suite "Serialization": - kind: deD""", output + test "Load object with ignored key": + let input = "[{x: 1, y: 2}, {x: 3, z: 4, y: 5}, {z: [1, 2, 3], x: 4, y: 5}]" + var result: seq[WithIgnoredField] + load(input, result) + assert result.len == 3 + assert result[0].x == 1 + assert result[0].y == 2 + assert result[1].x == 3 + assert result[1].y == 5 + assert result[2].x == 4 + assert result[2].y == 5 + + test "Load object with ignored key - unknown field": + let input = "{x: 1, y: 2, zz: 3}" + var result: WithIgnoredField + expectConstructionError(1, 16, "While constructing WithIgnoredField: Unknown field: \"zz\""): + load(input, result) + test "Dump cyclic data structure": var a = newNode("a") diff --git a/yaml/serialization.nim b/yaml/serialization.nim index 99c559a..31a5820 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -545,10 +545,13 @@ let newIdentNode(":defaultBitvector") defaultValueGetter {.compileTime.} = newIdentNode(":defaultValueGetter") + ignoredKeyListProc {.compileTime.} = + newIdentNode(":ignoredKeyList") var transientVectors {.compileTime.} = newSeq[set[int16]]() defaultVectors {.compileTime.} = newSeq[set[int16]]() + ignoredKeyLists {.compileTime.} = newSeq[seq[string]]() proc addDefaultOr(tName: string, i: int, o: NimNode, field, elseBranch, defaultValues: NimNode): NimNode {.compileTime.} = @@ -570,7 +573,6 @@ proc checkMissing(s: NimNode, t: typedesc, tName: string, field: NimNode, newNimNode(nnkRaiseStmt).add(newCall( bindSym("constructionError"), s, newLit("While constructing " & tName & ": Missing field: " & escape($field)))), defaultValues))) - echo result.repr proc markAsFound(i: int, matched: NimNode): NimNode {.compileTime.} = newAssignment(newNimNode(nnkBracketExpr).add(matched, newLit(i)), @@ -722,6 +724,13 @@ proc isVariantObject(t: typedesc): bool {.compileTime.} = if child.kind == nnkRecCase: return true return false +macro injectIgnoredKeyList(t: typedesc, ident: untyped): typed = + result = quote do: + when compiles(`ignoredKeyListProc`(`t`)): + const `ident` = ignoredKeyLists[`ignoredKeyListproc`(`t`)] + else: + const `ident` = newSeq[string]() + proc constructObject*[O: object|tuple]( s: var YamlStream, c: ConstructionContext, result: var O) {.raises: [YamlConstructionError, YamlStreamError].} = @@ -736,9 +745,8 @@ proc constructObject*[O: object|tuple]( raise s.constructionError("While constructing " & typetraits.name(O) & ": Expected " & $startKind & ", got " & $e.kind) when isVariantObject(O): reset(result) # make discriminants writeable + injectIgnoredKeyList(O, ignoredKeyList) while s.peek.kind != endKind: - # todo: check for duplicates in input and raise appropriate exception - # also todo: check for missing items and raise appropriate exception e = s.next() when isVariantObject(O): if e.kind != yamlStartMap: @@ -764,7 +772,17 @@ proc constructObject*[O: object|tuple]( raise s.constructionError("While constructing " & typetraits.name(O) & ": Unknown field: " & escape(name)) else: - constructFieldValue(O, tIndex, s, c, name, result, matched) + if name notin ignoredKeyList: + constructFieldValue(O, tIndex, s, c, name, result, matched) + else: + e = s.next() + var depth = int(e.kind in {yamlStartMap, yamlStartSeq}) + while depth > 0: + case s.next().kind + of yamlStartMap, yamlStartSeq: inc(depth) + of yamlEndMap, yamlEndSeq: dec(depth) + of yamlScalar: discard + else: internalError("Unexpected event kind.") when isVariantObject(O): e = s.next() if e.kind != yamlEndMap: @@ -1336,4 +1354,15 @@ macro setDefaultValue*(t: typedesc, field: untyped, value: typed): typed = static: `defaultValueGetter`(`t`).`field` = `value` defaultVectors[`defaultBitvectorProc`(`t`)].incl(getFieldIndex(`t`, `field`)) - echo result.repr \ No newline at end of file + +macro ignoreInputKey*(t: typedesc, name: string{lit}): typed = + let nextIgnoredKeyList = ignoredKeyLists.len + result = quote do: + when not compiles(`ignoredKeyListProc`(`t`)): + proc `ignoredKeyListProc`*(t: typedesc[`t`]): int {.compileTime.} = + `nextIgnoredKeyList` + static: ignoredKeyLists.add(@[]) + when `name` in ignoredKeyLists[`ignoredKeyListProc`(`t`)]: + {.fatal: "Input key " & `name` & " is already ignored!".} + static: + ignoredKeyLists[`ignoredKeyListProc`(`t`)].add(`name`) \ No newline at end of file From 6a3d876f7eecfe53483594aa2b80dbed06e8b376 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Thu, 27 Oct 2016 18:57:32 +0200 Subject: [PATCH 22/26] Generate testing.html with rst * use same html template as other pages * Also improved .gitignore --- .gitignore | 2 ++ config.nims | 3 +- doc/style.css | 5 +++ doc/{testing.html => testing.txt} | 57 +++++++++---------------------- 4 files changed, 26 insertions(+), 41 deletions(-) rename doc/{testing.html => testing.txt} (75%) diff --git a/.gitignore b/.gitignore index 704e960..7629330 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ test/tlex test/tdom test/tserialization test/tjson +test/tparser test/yamlTestSuite test/tquickstart test/*.exe @@ -22,3 +23,4 @@ doc/rstPreproc doc/tmp.rst doc/**/code yaml-dev-kit +nimsuggest.log diff --git a/config.nims b/config.nims index 57572d2..5e34367 100644 --- a/config.nims +++ b/config.nims @@ -48,7 +48,8 @@ task documentation, "Generate documentation": exec r"nim rst2html -o:../docout/api.html tmp.rst" exec r"./rstPreproc -o:tmp.rst serialization.txt" exec r"nim rst2html -o:../docout/serialization.html tmp.rst" - exec "cp docutils.css style.css testing.html processing.svg ../docout" + exec r"nim rst2html -o:../docout/testing.html testing.txt" + exec "cp docutils.css style.css processing.svg ../docout" exec r"nim doc2 -o:docout/yaml.html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/`git log -n 1 --format=%H` yaml" for file in listFiles("yaml"): let packageName = file[5..^5] diff --git a/doc/style.css b/doc/style.css index 3be9e52..ec7d977 100644 --- a/doc/style.css +++ b/doc/style.css @@ -108,6 +108,11 @@ dt a:before { visibility: hidden; } +#testingground { + margin-left: -50px; + margin-right: -50px; +} + #testingground textarea { width: 100%; height: 100%; diff --git a/doc/testing.html b/doc/testing.txt similarity index 75% rename from doc/testing.html rename to doc/testing.txt index 99ac2b8..a6a3b01 100644 --- a/doc/testing.html +++ b/doc/testing.txt @@ -1,32 +1,11 @@ - - - - - NimYAML - Testing Ground +============== +Testing Ground +============== - - - - - - - -
- NimYAML - Home - Testing Ground - Docs: - Overview - Serialization - Module yaml -
-
-
-

Testing Ground

-

Input is being processed on the fly by a friendly web service and - Output is updated as you type.

-
-
+Input is being processed on the fly by a friendly web service and output is +updated as you type. + +.. raw:: html
@@ -39,16 +18,16 @@
+ - test some + - {YAML: here} + - foo: bar + ? [1, 2, 3] + : !!str "string" + - + ? &a anchor + : !!bool yes + ? reference to anchor + : *a
@@ -139,5 +118,3 @@ } parse(); - - From 05c4aa733c67e04467536929b9c687caa279002f Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Tue, 1 Nov 2016 12:40:53 +0100 Subject: [PATCH 23/26] Documentation update * Added schema documentation * Documented setDefaultValue * Updated some tag URIs --- config.nims | 3 +- doc/schema.rst | 139 +++++++++++++++++++++++++++++++ doc/serialization.txt | 25 +++++- doc/style.css | 7 +- doc/{testing.txt => testing.rst} | 0 nimdoc.cfg | 10 ++- 6 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 doc/schema.rst rename doc/{testing.txt => testing.rst} (100%) diff --git a/config.nims b/config.nims index 5e34367..da29fc0 100644 --- a/config.nims +++ b/config.nims @@ -48,7 +48,8 @@ task documentation, "Generate documentation": exec r"nim rst2html -o:../docout/api.html tmp.rst" exec r"./rstPreproc -o:tmp.rst serialization.txt" exec r"nim rst2html -o:../docout/serialization.html tmp.rst" - exec r"nim rst2html -o:../docout/testing.html testing.txt" + exec r"nim rst2html -o:../docout/testing.html testing.rst" + exec r"nim rst2html -o:../docout/schema.html schema.rst" exec "cp docutils.css style.css processing.svg ../docout" exec r"nim doc2 -o:docout/yaml.html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/`git log -n 1 --format=%H` yaml" for file in listFiles("yaml"): diff --git a/doc/schema.rst b/doc/schema.rst new file mode 100644 index 0000000..b838c78 --- /dev/null +++ b/doc/schema.rst @@ -0,0 +1,139 @@ +==================== +Serialization Schema +==================== + +This document details the existing mappings in NimYAML from Nim types to YAML +tags. Throughout this document, there are two *tag shorthands* being used: + +========= ========================= +Shorthand Expansion +========= ========================= +``!!`` ``tag:yaml.org,2002:`` +``!n!`` ``tag:nimyaml.org,2016:`` +========= ========================= + +The first one is defined by the YAML specification and is used for types from +the YAML failsafe, JSON or core schema. The second one is defined by NimYAML and +is used for types from the Nim standard library. + +The YAML tag system has no understanding of generics. This means that NimYAML +must map every generic type instance to a YAML tag that describes that exact +type instance. For example, a ``seq[string]`` is mapped to the tag +``!n!system:seq(tag:yaml.org;2002:string)``. + +As you can see, the expanded tag handle of the generic type parameter is added +to the tag of the generic type. To be compliant with the YAML spec, the +following modifications are made: + +* Any exclamation marks are removed from the expanded tag. An exclamation mark + may only occur at the beginning of the tag as defined by the YAML spec. +* Any commas are replaces by semicolons, because they may not occur in a tag + apart from within the tag handle expansion. + +If a type takes multiple generic parameters, the tag handles are separated by +semicolons within the parentheses. Note that this does not guarantee unique tag +handles for every type, but it is currently seen as good enough. + +Note that user-defined generic types are currently not officially supported by +NimYAML. Only the generic collection types explicitly listed here use this +mechanism for crafting YAML tags. + +Scalar Types +============ + +The following table defines all non-composed, atomar types that are mapped to +YAML types by NimYAML. + +========= =========================================================== +Nim type YAML tag +========= =========================================================== +char ``!n!system:char`` +string ``!!string`` (or ``!n!nil:string`` if nil) +int ``!n!system:int32`` (independent on target architecture) +int8 ``!n!system:int8`` +int16 ``!n!system:int16`` +int32 ``!n!system:int32`` +int64 ``!n!system:int64`` +uint ``!n!system:uint32`` (independent from target architecture) +uint8 ``!n!system:uint8`` +uint16 ``!n!system:uint16`` +uint32 ``!n!system:uint32`` +uint64 ``!n!system:uint64`` +float ``!n!system:float64`` +float32 ``!n!system:float32`` +float64 ``!n!system:float64`` +bool ``!!bool`` +========= =========================================================== + +Apart from these standard library types, NimYAML also supports all enum types +as scalar types. They will be serialized to their string representation. + +Apart from the types listed here and enum tyes, no atomar types are supported. + +Collection Types +================ + +Collection types in Nim are typically generic. As such, they take their +contained types as parameters inside parentheses as explained above. The +following types are supported: + +============ ============================================================ ================================ +Nim type YAML tag YAML structure +============ ============================================================ ================================ +array ``!n!system:array(?;?)`` (first parameter like ``0..5``) sequence +seq ``!n!system:seq(?)`` (or ``!n!nil:seq`` if nil) sequence +set ``!n!system:set(?)`` sequence +Table ``!n!tables:Table(?;?)`` mapping +OrderedTable ``!n!tables:OrderedTable(?;?)`` sequence of single-pair mappings +============ ============================================================ ================================ + +Standard YAML Types +=================== + +NimYAML does not support all types defined in the YAML specification, **not even +those of the failsafe schema**. The reason is that the failsafe schema is +designed for dynamic type systems where a sequence can contain arbitrarily typed +values. This is not fully translatable into a static type system. NimYAML does +support some mechanisms to make working with heterogeneous collection structures +easier, see `Serialization Overview `_. + +Note that because the specification only defines that an implementation *should* +implement the failsafe schema, NimYAML is still compliant; it has valid reasons +not to implement the schema. + +This is a full list of all types defined in the YAML specification or the +`YAML type registry `_. It gives an overview of which +types are supported by NimYAML, which may be supported in the future and which +will never be supported. + +=============== ============================================ +YAML type Status +=============== ============================================ +``!!map`` Cannot be supported +``!!omap`` Cannot be supported +``!!pairs`` Cannot be supported +``!!set`` Cannot be supported +``!!seq`` Cannot be supported +``!!binary`` Currently not supported +``!!bool`` Maps to Nim's ``bool`` type +``!!float`` Not supported (user can choose) +``!!int`` Not supported (user can choose) +``!!merge`` Not supported and unlikely to be implemented +``!!null`` Used for reference types that are ``nil`` +``!!str`` Maps to Nim's ``string`` type +``!!timestamp`` Currently not supported +``!!value`` Not supported and unlikely to be implemented +``!!yaml`` Not supported and unlikely to be implemented +=============== ============================================ + +``!!int`` and ``!!float`` are not supported out of the box to let the user +choose where to map them (for example, ``!!int`` may map to ``int32`` or +``int64``, or the the generic ``int`` whose size is platform-dependent). If one +wants to use ``!!int``or ``!!float``, the process is to create a ``distinct`` +type derived from the desired base type and then set its tag using +``setTagUri``. + +``!!merge`` and ``!!value`` are not supported because the semantics of these +types would make a multi-pass loading process necessary and if one takes the +tag system seriously, ``!!merge`` can only be used with YAML's collection types, +which, as explained above, cannot be supported. \ No newline at end of file diff --git a/doc/serialization.txt b/doc/serialization.txt index f19ba54..6c36916 100644 --- a/doc/serialization.txt +++ b/doc/serialization.txt @@ -36,7 +36,8 @@ Supported Types NimYAML supports a growing number of types of Nim's ``system`` module and standard library, and it also supports user-defined object, tuple and enum types -out of the box. +out of the box. A complete list of explicitly supported types is available in +`Schema `_. **Important**: NimYAML currently does not support polymorphism. This may be added in the future. @@ -105,7 +106,7 @@ possible to dump and load cyclic data structures without further configuration. It is possible for reference types to hold a ``nil`` value, which will be mapped to the ``!!null`` YAML scalar type. -Pointer types are not supported because it seems dangerous to automatically +``ptr`` types are not supported because it seems dangerous to automatically allocate memory which the user must then manually deallocate. User Defined Types @@ -234,7 +235,7 @@ otherwise, it would be loaded as ``nil``. As you might have noticed in the example above, the YAML tag of a ``seq`` depends on its generic type parameter. The same applies to ``Table``. So, a table that maps ``int8`` to string sequences would be presented with the tag -``!nim:tables:Table(nim:system:int8,nim:system:seq(tag:yaml.org,2002:string))``. +``!n!tables:Table(tag:nimyaml.org,2016:int8,tag:nimyaml.org,2016:system:seq(tag:yaml.org,2002:string))``. These tags are generated on the fly based on the types you instantiate ``Table`` or ``seq`` with. @@ -274,6 +275,24 @@ Example: markAsTransient(MyObject, temporary) +Default Values +-------------- + +When you load YAML that has been written by a human, you might want to allow the +user to omit certain fields, which should then be filled with a default value. +You can do that like this: + +.. code-block:: nim + + type MyObject: object + required: string + optional: string + + setDefaultValue(MyObject, optional, "default value") + +Whenever ``MyObject`` now is loaded and the input stream does not contain the +field ``optional``, that field will be set to the value ``"default value"``. + Customize Serialization ======================= diff --git a/doc/style.css b/doc/style.css index ec7d977..513492f 100644 --- a/doc/style.css +++ b/doc/style.css @@ -54,12 +54,15 @@ header span:hover > ul { } header span ul a { - font-size: smaller; - font-family: "Source Code Pro", Menlo, "Courier New", Courier, monospace; padding: 0 10px; line-height: 40px; } +header span ul.monospace a { + font-size: smaller; + font-family: "Source Code Pro", Menlo, "Courier New", Courier, monospace; +} + header a:link, header a:visited { background: inherit; diff --git a/doc/testing.txt b/doc/testing.rst similarity index 100% rename from doc/testing.txt rename to doc/testing.rst diff --git a/nimdoc.cfg b/nimdoc.cfg index fcea78a..0ab14c9 100644 --- a/nimdoc.cfg +++ b/nimdoc.cfg @@ -110,10 +110,16 @@ doc.file = """ Testing Ground Docs: Overview - Serialization + + Serialization + + Modules -
    +
    • yaml
    • yaml.dom
    • yaml.hints
      • From 64f68ae1afc499db40fe7c8f5b69a1c3b8251496 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Tue, 8 Nov 2016 21:13:01 +0100 Subject: [PATCH 24/26] Implemented !!timestamp --- test/tserialization.nim | 10 +++- yaml/hints.nim | 101 +++++++++++++++++++++++++++++++++++----- yaml/serialization.nim | 70 +++++++++++++++++++++++++++- 3 files changed, 167 insertions(+), 14 deletions(-) diff --git a/test/tserialization.nim b/test/tserialization.nim index c089926..1261132 100644 --- a/test/tserialization.nim +++ b/test/tserialization.nim @@ -5,7 +5,7 @@ # distribution, for details about the copyright. import "../yaml" -import unittest, strutils, streams, tables +import unittest, strutils, streams, tables, times type MyTuple = tuple @@ -208,6 +208,14 @@ suite "Serialization": var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual yamlDirs & "\n!n!nil:string \"\"", output + test "Load timestamps": + let input = "[2001-12-15T02:59:43.1Z, 2001-12-14t21:59:43.10-05:00, 2001-12-14 21:59:43.10-5]" + var result: seq[Time] + load(input, result) + assert result.len() == 3 + # currently, there is no good way of checking the result content, because + # the parsed Time may have any timezone offset. + test "Load string sequence": let input = newStringStream(" - a\n - b") var result: seq[string] diff --git a/yaml/hints.nim b/yaml/hints.nim index 0892787..b3b6e4a 100644 --- a/yaml/hints.nim +++ b/yaml/hints.nim @@ -33,10 +33,11 @@ type ## ``yTypeBoolTrue`` ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON`` ## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF`` ## ``yTypeNull`` ``~ | null | Null | NULL`` + ## ``yTypeTimestamp`` see `here `_. ## ``yTypeUnknown`` ``*`` ## ================== ========================= yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue, - yTypeBoolFalse, yTypeNull, yTypeUnknown + yTypeBoolFalse, yTypeNull, yTypeUnknown, yTypeTimestamp YamlTypeHintState = enum ythInitial, @@ -59,7 +60,16 @@ type ythPointLowerIN, ythPointLowerN, ythPointLowerNA, - ythMinus, yth0, ythInt, ythDecimal, ythNumE, ythNumEPlusMinus, ythExponent + ythMinus, yth0, ythInt1, ythInt1Zero, ythInt2, ythInt2Zero, ythInt3, + ythInt3Zero, ythInt4, ythInt4Zero, ythInt, + ythDecimal, ythNumE, ythNumEPlusMinus, ythExponent, + + ythYearMinus, ythMonth1, ythMonth2, ythMonthMinus, ythMonthMinusNoYmd, + ythDay1, ythDay1NoYmd, ythDay2, ythDay2NoYmd, + ythAfterDayT, ythAfterDaySpace, ythHour1, ythHour2, ythHourColon, + ythMinute1, ythMinute2, ythMinuteColon, ythSecond1, ythSecond2, ythFraction, + ythAfterTimeSpace, ythAfterTimeZ, ythAfterTimePlusMinus, ythTzHour1, + ythTzHour2, ythTzHourColon, ythTzMinute1, ythTzMinute2 macro typeHintStateMachine(c: untyped, content: untyped): typed = yAssert content.kind == nnkStmtList @@ -101,20 +111,78 @@ template advanceTypeHint(ch: char) {.dirty.} = typeHintStateMachine ch: of '~': ythInitial => ythNULL of '.': - [yth0, ythInt] => ythDecimal + [yth0, ythInt1, ythInt2, ythInt3, ythInt4, ythInt] => ythDecimal [ythInitial, ythMinus] => ythPoint - of '+': ythNumE => ythNumEPlusMinus + ythSecond2 => ythFraction + of '+': + ythNumE => ythNumEPlusMinus + [ythFraction, ythSecond2] => ythAfterTimePlusMinus of '-': - ythInitial => ythMinus - ythNumE => ythNumEPlusMinus + ythInitial => ythMinus + ythNumE => ythNumEPlusMinus + [ythInt4, ythInt4Zero] => ythYearMinus + ythMonth1 => ythMonthMinusNoYmd + ythMonth2 => ythMonthMinus + [ythFraction, ythSecond2] => ythAfterTimePlusMinus + of ':': + [ythHour1, ythHour2] => ythHourColon + ythMinute2 => ythMinuteColon + [ythTzHour1, ythTzHour2] => ythTzHourColon of '0': - [ythInitial, ythMinus] => yth0 + ythInitial => ythInt1Zero + ythMinus => yth0 [ythNumE, ythNumEPlusMinus] => ythExponent - [ythInt, ythDecimal, ythExponent] => nil + ythInt1 => ythInt2 + ythInt1Zero => ythInt2Zero + ythInt2 => ythInt3 + ythInt2Zero => ythInt3Zero + ythInt3 => ythInt4 + ythInt3Zero => ythInt4Zero + ythInt4 => ythInt + ythYearMinus => ythMonth1 + ythMonth1 => ythMonth2 + ythMonthMinus => ythDay1 + ythMonthMinusNoYmd => ythDay1NoYmd + ythDay1 => ythDay2 + ythDay1NoYmd => ythDay2NoYmd + [ythAfterDaySpace, ythAfterDayT] => ythHour1 + ythHour1 => ythHour2 + ythHourColon => ythMinute1 + ythMinute1 => ythMinute2 + ythMinuteColon => ythSecond1 + ythSecond1 => ythSecond2 + ythAfterTimePlusMinus => ythTzHour1 + ythTzHour1 => ythTzHour2 + ythTzHourColon => ythTzMinute1 + ythTzMinute1 => ythTzMinute2 + [ythInt, ythDecimal, ythExponent, ythFraction] => nil of '1'..'9': - [ythInitial, ythMinus] => ythInt + ythInitial => ythInt1 + ythInt1 => ythInt2 + ythInt1Zero => ythInt2Zero + ythInt2 => ythInt3 + ythInt2Zero => ythInt3Zero + ythInt3 => ythInt4 + ythInt3Zero => ythInt4Zero + [ythInt4, ythMinus] => ythInt [ythNumE, ythNumEPlusMinus] => ythExponent - [ythInt, ythDecimal, ythExponent] => nil + ythYearMinus => ythMonth1 + ythMonth1 => ythMonth2 + ythMonthMinus => ythDay1 + ythMonthMinusNoYmd => ythDay1NoYmd + ythDay1 => ythDay2 + ythDay1NoYmd => ythDay2NoYmd + [ythAfterDaySpace, ythAfterDayT] => ythHour1 + ythHour1 => ythHour2 + ythHourColon => ythMinute1 + ythMinute1 => ythMinute2 + ythMinuteColon => ythSecond1 + ythSecond1 => ythSecond2 + ythAfterTimePlusMinus => ythTzHour1 + ythTzHour1 => ythTzHour2 + ythTzHourColon => ythTzMinute1 + ythTzMinute1 => ythTzMinute2 + [ythInt, ythDecimal, ythExponent, ythFraction] => nil of 'a': ythF => ythLowerFA ythPointN => ythPointNA @@ -174,7 +242,9 @@ template advanceTypeHint(ch: char) {.dirty.} = of 'S': ythFAL => ythFALS ythYE => ythYES - of 't', 'T': ythInitial => ythT + of 't', 'T': + ythInitial => ythT + [ythDay1, ythDay2, ythDay1NoYmd, ythDay2NoYmd] => ythAfterDayT of 'u': ythN => ythLowerNU ythLowerTR => ythLowerTRU @@ -182,6 +252,11 @@ template advanceTypeHint(ch: char) {.dirty.} = ythN => ythNU ythTR => ythTRU of 'y', 'Y': ythInitial => ythY + of 'Z': [ythSecond2, ythFraction, ythAfterTimeSpace] => ythAfterTimeZ + of ' ', '\t': + [ythSecond2, ythFraction] => ythAfterTimeSpace + [ythDay1, ythDay2, ythDay1NoYmd, ythDay2NoYmd] => ythAfterDaySpace + [ythAfterTimeSpace, ythAfterDaySpace] => nil proc guessType*(scalar: string): TypeHint {.raises: [].} = ## Parse scalar string according to the RegEx table documented at @@ -192,8 +267,10 @@ proc guessType*(scalar: string): TypeHint {.raises: [].} = of ythNULL: result = yTypeNull of ythTRUE, ythON, ythYES, ythY: result = yTypeBoolTrue of ythFALSE, ythOFF, ythNO, ythN: result = yTypeBoolFalse - of ythInt, yth0: result = yTypeInteger + of ythInt1, ythInt2, ythInt3, ythInt4, ythInt, yth0: result = yTypeInteger of ythDecimal, ythExponent: result = yTypeFloat of ythPointINF: result = yTypeFloatInf of ythPointNAN: result = yTypeFloatNaN + of ythDay2, ythSecond2, ythFraction, ythAfterTimeZ, ythTzHour1, ythTzHour2, + ythTzMinute1, ythTzMinute2: result = yTypeTimestamp else: result = yTypeUnknown diff --git a/yaml/serialization.nim b/yaml/serialization.nim index 31a5820..d1365d9 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -16,7 +16,7 @@ ## type. Please consult the serialization guide on the NimYAML website for more ## information. -import tables, typetraits, strutils, macros, streams +import tables, typetraits, strutils, macros, streams, times import parser, taglib, presenter, stream, ../private/internal, hints export stream # *something* in here needs externally visible `==`(x,y: AnchorId), @@ -321,6 +321,72 @@ proc representObject*(value: char, ts: TagStyle, c: SerializationContext, ## represents a char value as YAML scalar c.put(scalarEvent("" & value, tag, yAnchorNone)) +proc yamlTag*(T: typedesc[Time]): TagId {.inline, raises: [].} = yTagTimestamp + +proc constructObject*(s: var YamlStream, c: ConstructionContext, + result: var Time) + {.raises: [YamlConstructionError, YamlStreamError].} = + constructScalarItem(s, item, Time): + if guessType(item.scalarContent) == yTypeTimestamp: + var + tmp = newStringOfCap(60) + pos = 8 + c: char + while pos < item.scalarContent.len(): + c = item.scalarContent[pos] + if c in {' ', '\t', 'T', 't'}: break + inc(pos) + if pos == item.scalarContent.len(): + tmp.add(item.scalarContent) + tmp.add("T00:00:00+00:00") + else: + tmp.add(item.scalarContent[0 .. pos - 1]) + if c in {' ', '\t'}: + while true: + inc(pos) + c = item.scalarContent[pos] + if c notin {' ', '\t'}: break + else: inc(pos) + tmp.add("T") + let timeStart = pos + inc(pos, 7) + var fractionStart = -1 + while pos < item.scalarContent.len(): + c = item.scalarContent[pos] + if c in {'+', '-', 'Z', ' ', '\t'}: break + elif c == '.': fractionStart = pos + inc(pos) + if fractionStart == -1: + tmp.add(item.scalarContent[timeStart .. pos - 1]) + else: + tmp.add(item.scalarContent[timeStart .. fractionStart - 1]) + if c in {'Z', ' ', '\t'}: tmp.add("+00:00") + else: + tmp.add(c) + inc(pos) + let tzStart = pos + inc(pos) + if pos < item.scalarContent.len() and item.scalarContent[pos] != ':': + inc(pos) + if pos - tzStart == 1: tmp.add('0') + tmp.add(item.scalarContent[tzStart .. pos - 1]) + if pos == item.scalarContent.len(): tmp.add(":00") + elif pos + 2 == item.scalarContent.len(): + tmp.add(":0") + tmp.add(item.scalarContent[pos + 1]) + else: + tmp.add(item.scalarContent[pos .. pos + 2]) + let info = tmp.parse("yyyy-M-d'T'H-mm-sszzz") + result = info.toTime() + else: + raise s.constructionError("Not a parsable timestamp: " & + escape(item.scalarContent)) + +proc representObject*(value: Time, ts: TagStyle, c: SerializationContext, + tag: TagId) {.raises: [ValueError].} = + let tmp = value.getGMTime() + c.put(scalarEvent(tmp.format("yyyy-MM-dd'T'HH:mm:ss'Z'"))) + proc yamlTag*[I](T: typedesc[seq[I]]): TagId {.inline, raises: [].} = let uri = nimTag("system:seq(" & safeTagUri(yamlTag(I)) & ')') result = lazyLoadTag(uri) @@ -984,6 +1050,8 @@ proc constructChild*[T](s: var YamlStream, c: ConstructionContext, raise s.constructionError("not implemented!") of yTypeUnknown: possibleTagIds.add(yamlTag(string)) + of yTypeTimestamp: + possibleTagIds.add(yamlTag(Time)) of yTagExclamationMark: possibleTagIds.add(yamlTag(string)) else: From 86845f9bd5feeb9335a8739af52af28afdeec999 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Tue, 8 Nov 2016 21:15:14 +0100 Subject: [PATCH 25/26] Documented !!timestamp --- doc/schema.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/schema.rst b/doc/schema.rst index b838c78..8b22c99 100644 --- a/doc/schema.rst +++ b/doc/schema.rst @@ -63,6 +63,7 @@ float ``!n!system:float64`` float32 ``!n!system:float32`` float64 ``!n!system:float64`` bool ``!!bool`` +Time ``!!timestamp`` ========= =========================================================== Apart from these standard library types, NimYAML also supports all enum types @@ -121,7 +122,7 @@ YAML type Status ``!!merge`` Not supported and unlikely to be implemented ``!!null`` Used for reference types that are ``nil`` ``!!str`` Maps to Nim's ``string`` type -``!!timestamp`` Currently not supported +``!!timestamp`` Maps to Nim's ``Time`` type ``!!value`` Not supported and unlikely to be implemented ``!!yaml`` Not supported and unlikely to be implemented =============== ============================================ From 2ba08370eba5c03ae6663d13dc49907142ae2125 Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Tue, 8 Nov 2016 21:29:59 +0100 Subject: [PATCH 26/26] Version 0.8.0 --- CHANGELOG.md | 22 ++++++++++++++++++++++ yaml.nimble | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a77ab..290d445 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +### 0.8.0 + +Features: + + * NimYAML now has a global tag URI prefix for Nim types, + `tag:nimyaml.org,2016:`. This prefix is denoted by the custom tag handle + `!n!`. + * Support arbitrary tag handles. + * Added ability to mark object and tuple fields as transient. + * Added ability to set a default value for object fields. + * Added ability to ignore key-value pairs in the input when loading object + values. + * Support `!!timestamp` by parsing it to `Time` from module `times`. + +Bugfixes: + + * Fixed a bug concerning duplicate TagIds for different tags in the + `serializationTagLibrary` + * Convert commas in tag URIs to semicolons when using a tag URI as generic + parameter to another one, because commas after the tag handle are interpreted + as flow separators. + ### 0.7.0 Features: diff --git a/yaml.nimble b/yaml.nimble index 2b9e26e..ceb5b6f 100644 --- a/yaml.nimble +++ b/yaml.nimble @@ -1,6 +1,6 @@ # Package -version = "0.7.0" +version = "0.8.0" author = "Felix Krause" description = "YAML 1.2 implementation for Nim" license = "MIT"