diff --git a/.gitignore b/.gitignore index d8eb3ab..2b7f03b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ nimcache test/tests -test/parsing -test/lexing -test/serializing -test/constructingJson +test/tlex +test/tdom +test/tserialization +test/tjson test/yamlTestSuite test/*.exe test/*.pdb diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..0347dd3 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,48 @@ +# Copied from https://github.com/nim-lang/Nim/wiki/TravisCI +language: c +env: + # Build and test against the master and devel branches of Nim + #- BRANCH=master TODO! does not work currently, waiting for 0.15.0 + - BRANCH=devel +compiler: + # Build and test using both gcc and clang + - gcc + - clang +matrix: + allow_failures: + # Ignore failures when building against the devel Nim branch + # - env: BRANCH=devel TODO! ignore those once stable Nim can build NimYAML + fast_finish: true +install: + - | + if [ ! -x nim-$BRANCH/bin/nim ]; then + git clone -b $BRANCH --depth 1 git://github.com/nim-lang/nim nim-$BRANCH/ + cd nim-$BRANCH + git clone -b $BRANCH --depth 1 git://github.com/nim-lang/csources csources/ + cd csources + sh build.sh + cd .. + rm -rf csources + bin/nim c koch + ./koch boot -d:release + else + cd nim-$BRANCH + git fetch origin + if ! git merge FETCH_HEAD | grep "Already up-to-date"; then + bin/nim c koch + ./koch boot -d:release + fi + fi + cd .. +before_script: + - export PATH="nim-$BRANCH/bin${PATH:+:$PATH}" +script: + - nim tests --verbosity:0 + - nim yamlTestSuite --verbosity:0 +cache: + directories: + - nim-master + - nim-devel +branches: + except: + - gh-pages \ No newline at end of file diff --git a/README.md b/README.md index 8dd7da3..4d5ba7a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # NimYAML - YAML implementation for Nim +[![Build Status](https://travis-ci.org/flyx/NimYAML.svg?branch=devel)](https://travis-ci.org/flyx/NimYAML) + NimYAML is a pure Nim YAML implementation without any dependencies other than Nim's standard library. It enables you to serialize Nim objects to a YAML stream and back. It also provides a low-level event-based API, and a document object diff --git a/bench/common.nim b/bench/commonBench.nim similarity index 100% rename from bench/common.nim rename to bench/commonBench.nim diff --git a/bench/jsonBench.nim b/bench/jsonBench.nim index 06b2867..d187e59 100644 --- a/bench/jsonBench.nim +++ b/bench/jsonBench.nim @@ -1,15 +1,15 @@ -import "../yaml", common +import "../yaml", commonBench from nimlets_yaml import objKind -import math, strutils, stopwatch, terminal, algorithm +import math, strutils, stopwatch, terminal, algorithm, random, json -proc cmp(left, right: clock): int = cmp(left.nanoseconds(), right.nanoseconds()) +proc cmp(left, right: clock): int = cmp(left.nanoseconds(), right.nanoseconds()) type ObjectKind = enum otMap, otSequence - + Level = tuple kind: ObjectKind len: int @@ -33,10 +33,10 @@ proc genString(maxLen: int): string = proc genJsonString(size: int, maxStringLen: int): string = ## Generates a random JSON string. ## size is in KiB, mayStringLen in characters. - + randomize(size * maxStringLen) result = "{" - + let targetSize = size * 1024 var indentation = 2 @@ -44,13 +44,13 @@ proc genJsonString(size: int, maxStringLen: int): string = curSize = 1 justOpened = true levels.add((kind: otMap, len: 0)) - + while levels.len > 0: let objectCloseProbability = float(levels[levels.high].len + levels.high) * 0.025 closeObject = random(1.0) <= objectCloseProbability - + if (closeObject and levels.len > 1) or curSize > targetSize: indentation -= 2 if justOpened: @@ -67,9 +67,9 @@ proc genJsonString(size: int, maxStringLen: int): string = curSize += 1 discard levels.pop() continue - + levels[levels.high].len += 1 - + if justOpened: justOpened = false result.add("\x0A") @@ -79,7 +79,7 @@ proc genJsonString(size: int, maxStringLen: int): string = result.add(",\x0A") result.add(repeat(' ', indentation)) curSize += indentation + 2 - + case levels[levels.high].kind of otMap: let key = genString(maxStringLen) @@ -88,12 +88,12 @@ proc genJsonString(size: int, maxStringLen: int): string = curSize += key.len + 2 of otSequence: discard - + let objectValueProbability = 0.8 / float(levels.len * levels.len) generateObjectValue = random(1.0) <= objectValueProbability - + if generateObjectValue: let objectKind = if random(2) == 0: otMap else: otSequence case objectKind @@ -126,7 +126,7 @@ proc genJsonString(size: int, maxStringLen: int): string = discard else: discard - + result.add(s) curSize += s.len @@ -138,7 +138,7 @@ var json100k = genJsonString(100, 32) tagLib = initCoreTagLibrary() parser = newYamlParser(initCoreTagLibrary()) - + block: multibench(cJson1k, 100): let res = parseJson(json1k) @@ -152,30 +152,21 @@ block: block: multibench(cJson100k, 100): let res = parseJson(json100k) - assert res.kind == JObject + assert res.kind == JObject block: multibench(cYaml1k, 100): - var - s = newStringStream(json1k) - events = parser.parse(s) - let res = constructJson(events) + let res = loadToJson(json1k) assert res[0].kind == JObject block: multibench(cYaml10k, 100): - var - s = newStringStream(json10k) - events = parser.parse(s) - let res = constructJson(events) + let res = loadToJson(json10k) assert res[0].kind == JObject block: multibench(cYaml100k, 100): - var - s = newStringStream(json100k) - events = parser.parse(s) - let res = constructJson(events) + let res = loadToJson(json100k) assert res[0].kind == JObject block: diff --git a/bench/yamlBench.nim b/bench/yamlBench.nim index 88fbe3a..9cb2e6d 100644 --- a/bench/yamlBench.nim +++ b/bench/yamlBench.nim @@ -1,5 +1,5 @@ -import "../yaml", common -import math, strutils, stopwatch, terminal, algorithm +import "../yaml", commonBench +import math, strutils, stopwatch, terminal, algorithm, random, streams from nimlets_yaml import objKind @@ -32,17 +32,16 @@ proc genKey(): string = let c = random(26 + 26 + 10) if c < 26: result.add(char(c + 65)) elif c < 52: result.add(char(c + 97 - 26)) - else: result.add(char(c + 48 - 52)) + else: result.add(char(c + 48 - 52)) else: result = genString(31) & char(random(26) + 65) proc genYamlString(size: int, maxStringLen: int, style: PresentationStyle): string = ## Generates a random YAML string. ## size is in KiB, mayStringLen in characters. - + randomize(size * maxStringLen * ord(style)) - result = "{" - + let targetSize = size * 1024 var target = newStringStream() @@ -53,13 +52,13 @@ proc genYamlString(size: int, maxStringLen: int, levels.add((kind: yMapping, len: 0)) yield startDocEvent() yield startMapEvent() - + while levels.len > 0: let objectCloseProbability = float(levels[levels.high].len + levels.high) * 0.025 closeObject = random(1.0) <= objectCloseProbability - + if (closeObject and levels.len > 1) or curSize > targetSize: case levels[levels.high].kind of yMapping: yield endMapEvent() @@ -68,19 +67,19 @@ proc genYamlString(size: int, maxStringLen: int, curSize += 1 discard levels.pop() continue - + levels[levels.high].len += 1 if levels[levels.high].kind == yMapping: let key = genKey() yield scalarEvent(key) - + let objectValueProbability = 0.8 / float(levels.len * levels.len) generateObjectValue = random(1.0) <= objectValueProbability hasTag = random(2) == 0 var tag = yTagQuestionMark - + if generateObjectValue: let objectKind = if random(3) == 0: ySequence else: yMapping case objectKind @@ -120,7 +119,7 @@ proc genYamlString(size: int, maxStringLen: int, if hasTag: tag = yTagNull else: discard else: discard - + yield scalarEvent(s, tag) curSize += s.len yield endDocEvent() @@ -128,32 +127,35 @@ proc genYamlString(size: int, maxStringLen: int, present(yStream, target, initExtendedTagLibrary(), defineOptions(style=style, outputVersion=ov1_1)) result = target.data - + var - cYaml1k, cYaml10k, cYaml100k, cLibYaml1k, cLibYaml10k, cLibYaml100k: int64 + cYaml1k, cYaml10k, cYaml100k, cLibYaml1k, cLibYaml10k, cLibYaml100k, + cYaml1m, cLibYaml1m: int64 yaml1k = genYamlString(1, 32, psDefault) yaml10k = genYamlString(10, 32, psDefault) yaml100k = genYamlString(100, 32, psDefault) + yaml1m = genYamlString(1000, 32, psDefault) tagLib = initExtendedTagLibrary() parser = newYamlParser(tagLib) block: multibench(cYaml1k, 100): - var s = newStringStream(yaml1k) - let res = loadDOM(s) + let res = loadDOM(yaml1k) assert res.root.kind == yMapping block: multibench(cYaml10k, 100): - var - s = newStringStream(yaml10k) - let res = loadDOM(s) + let res = loadDOM(yaml10k) assert res.root.kind == yMapping block: multibench(cYaml100k, 100): - var s = newStringStream(yaml100k) - let res = loadDOM(s) + let res = loadDOM(yaml100k) + assert res.root.kind == yMapping + +block: + multibench(cYaml1m, 2): + let res = loadDOM(yaml1m) assert res.root.kind == yMapping block: @@ -171,6 +173,11 @@ block: let res = nimlets_yaml.load(yaml100k) assert res[0].objKind == nimlets_yaml.YamlObjKind.Map +block: + multibench(cLibYaml1m, 2): + let res = nimlets_yaml.load(yaml1m) + assert res[0].objKind == nimlets_yaml.YamlObjKind.Map + proc writeResult(caption: string, num: int64) = styledWriteLine(stdout, resetStyle, caption, fgGreen, $num, resetStyle, "μs") @@ -188,4 +195,8 @@ writeResult "LibYAML: ", cLibYaml10k div 1000 setForegroundColor(fgWhite) writeStyled "100k input\n----------\n" writeResult "NimYAML: ", cYaml100k div 1000 -writeResult "LibYAML: ", cLibYaml100k div 1000 \ No newline at end of file +writeResult "LibYAML: ", cLibYaml100k div 1000 +setForegroundColor(fgWhite) +writeStyled "1m input\n---------\n" +writeResult "NimYAML: ", cYaml1m div 1000 +writeResult "LibYAML: ", cLibYaml1m div 1000 diff --git a/config.nims b/config.nims index 1021e14..0d2610f 100644 --- a/config.nims +++ b/config.nims @@ -21,6 +21,12 @@ task serializationTests, "Run serialization tests": task documentation, "Generate documentation": exec "mkdir -p docout" exec r"nim doc2 -o:docout/yaml.html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/`git log -n 1 --format=%H` yaml" + # bash! ah-ah \\ savior of the universe + for file in listFiles("yaml"): + let packageName = file[5..^5] + exec r"nim doc2 -o:docout/yaml." & packageName & + ".html --docSeeSrcUrl:https://github.com/flyx/NimYAML/blob/yaml/`git log -n 1 --format=%H` " & + file exec r"nim rst2html -o:docout/index.html doc/index.txt" exec r"nim rst2html -o:docout/api.html doc/api.txt" exec r"nim rst2html -o:docout/serialization.html doc/serialization.txt" @@ -28,10 +34,10 @@ task documentation, "Generate documentation": setCommand "nop" task bench, "Benchmarking": - --d:release --r --w:off --hints:off + --d:release setCommand "c", "bench/bench" task clean, "Remove all generated files": diff --git a/doc/api.txt b/doc/api.txt index d8ae049..9d38333 100644 --- a/doc/api.txt +++ b/doc/api.txt @@ -25,8 +25,8 @@ Intermediate Representation =========================== The base of all YAML processing with NimYAML is the -`YamlStream `_. This is basically an iterator over -`YamlStreamEvent `_ objects. Every proc that +`YamlStream `_. This is basically an iterator over +`YamlStreamEvent `_ objects. Every proc that represents a single stage of the loading or dumping process will either take a ``YamlStream`` as input or return a ``YamlStream``. Procs that implement the whole process in one step hide the ``YamlStream`` from the user. Every proc that @@ -45,16 +45,17 @@ Loading YAML ============ If you want to load YAML character data directly into a native Nim variable, you -can use `load `_. This is the easiest and recommended -way to load YAML data. The following paragraphs will explain the steps involved. +can use `load `_. This is the easiest and +recommended way to load YAML data. This section gives an overview about how +``load`` is implemented. It is absolutely possible to reimplement the loading +step using the low-level API. -For parsing, a `YamlParser `_ object is needed. This -object stores some state while parsing that may be useful for error reporting to -the user. The -`parse `_ proc implements the YAML processing -step of the same name. All syntax errors in the input character stream are -processed by ``parse``, which will raise a ``YamlParserError`` if it encounters -a syntax error. +For parsing, a `YamlParser `_ object is needed. +This object stores some state while parsing that may be useful for error +reporting to the user. The `parse `_ +proc implements the YAML processing step of the same name. All syntax errors in +the input character stream are processed by ``parse``, which will raise a +``YamlParserError`` if it encounters a syntax error. Transforming a ``YamlStream`` to a native YAML object is done via ``construct``. It skips the ``compose`` step for efficiency reasons. As Nim is @@ -67,19 +68,21 @@ Dumping YAML ============ Dumping is preferredly done with -`dump `_, -which serializes a native Nim variable to a character stream. Like ``load``, you -can use the steps involved separately. +`dump `_, +which serializes a native Nim variable to a character stream. As with ``load``, +the following paragraph describes how ``dump`` is implemented using the +low-level API. -You transform a variable into a ``YamlStream`` with -`represent `_. Depending on the -``AnchorStyle`` you specify, this will transform ``ref`` variables with multiple -instances into anchored elements and aliases (for ``asTidy`` and ``asAlways``) -or write the same element into all places it occurs (for ``asNone``). Be aware -that if you use ``asNone``, the value you serialize might not round-trip. +A Nim value is transformed into a ``YamlStream`` with +`represent `_. +Depending on the ``AnchorStyle`` you specify, this will transform ``ref`` +variables with multiple instances into anchored elements and aliases (for +``asTidy`` and ``asAlways``) or write the same element into all places it +occurs (for ``asNone``). Be aware that if you use ``asNone``, the value you +serialize might not round-trip. Transforming a ``YamlStream`` into YAML character data is done with -`present `_. +`present `_. You can choose from multiple presentation styles. ``psJson`` is not able to process some features of ``YamlStream`` s, the other styles support all features and are guaranteed to round-trip to the same ``YamlStream`` if you parse the @@ -90,11 +93,12 @@ The Document Object Model Much like XML, YAML also defines a *document object model*. If you cannot or do not want to load a YAML character stream to native Nim types, you can instead -load it into a ``YamlDocument``. This ``YamlDocument`` can also be serialized -into a YAML character stream. All tags will be preserved exactly as they are -when transforming from and to a ``YamlDocument``. The only important thing to -remember is that when a value has no tag, it will get the non-specific tag -``"!"`` for quoted scalars and ``"?"`` for all other nodes. +load it into a `YamlDocument `_. This +``YamlDocument`` can also be serialized into a YAML character stream. All tags +will be preserved exactly as they are when transforming from and to a +``YamlDocument``. The only important thing to remember is that when a value has +no tag, it will get the non-specific tag ``"!"`` for quoted scalars and ``"?"`` +for all other nodes. While tags are preserved, anchors will be resolved during loading and re-added during serialization. It is allowed for a ``YamlNode`` to occur multiple times diff --git a/doc/processing.svg b/doc/processing.svg index 734b5d1..a312738 100644 --- a/doc/processing.svg +++ b/doc/processing.svg @@ -68,18 +68,18 @@ - + Application YAML - + Native Nim Types - + Representation @@ -87,7 +87,7 @@ YamlDocument - + Serialization @@ -95,7 +95,7 @@ YamlStream - + Presentation @@ -104,87 +104,87 @@ - + represent - + - + serialize - + - + present - + construct - + - + compose - + - + parse - + - + represent - + construct - + - + dump - - dumpDOM + + dumpDOM - + loadDOM - + load - + \ No newline at end of file diff --git a/doc/serialization.txt b/doc/serialization.txt index 4197d9a..8835a55 100644 --- a/doc/serialization.txt +++ b/doc/serialization.txt @@ -8,10 +8,11 @@ Introduction NimYAML tries hard to make transforming YAML characters streams to native Nim types and vice versa as easy as possible. In simple scenarios, you might not need anything else than the two procs -`dump `_ and -`load `_. On the other side, the process should be as -customizable as possible to allow the user to tightly control how the generated -YAML character stream will look and how a YAML character stream is interpreted. +`dump `_ +and `load `_. On the other side, the process +should be as customizable as possible to allow the user to tightly control how +the generated YAML character stream will look and how a YAML character stream is +interpreted. An important thing to remember in NimYAML is that unlike in interpreted languages like Ruby, Nim cannot load a YAML character stream without knowing the @@ -42,7 +43,8 @@ added in the future. This also means that NimYAML is generally able to work with object, tuple and enum types defined in the standard library or a third-party library without -further configuration. +further configuration, given that all fields of the object are accessible at the +code point where NimYAML's facilities are invoked. Scalar Types ------------ @@ -113,7 +115,7 @@ For an object or tuple type to be directly usable with NimYAML, the following conditions must be met: - Every type contained in the object/tuple must be supported -- All fields of an object type must be accessible from the code position where +- All fields of an object type must be accessible from the code position where you call NimYAML. If an object has non-public member fields, it can only be processed in the module where it is defined. - The object may not have a generic parameter @@ -141,7 +143,7 @@ For example, this type: type AnimalKind = enum akCat, akDog - + Animal = object name: string case kind: AnimalKind @@ -175,7 +177,7 @@ list in order to load it: .. code-block:: nim import yaml - + type ContainerKind = enum ckInt, ckString, ckNone @@ -187,9 +189,9 @@ list in order to load it: strVal: string of ckNone: discard - + markAsImplicit(Container) - + var list: seq[Container] s = newFileStream("in.yaml") @@ -232,7 +234,7 @@ otherwise, it would be loaded as ``nil``. As you might have noticed in the example above, the YAML tag of a ``seq`` depends on its generic type parameter. The same applies to ``Table``. So, a table that maps ``int8`` to string sequences would be presented with the tag -``!nim:tables:Table(nim:system:int8,nim:system:seq(tag:yaml.org,2002:string))``. +``!nim:tables:Table(nim:system:int8,nim:system:seq(tag:yaml.org,2002:string))``. These tags are generated on the fly based on the types you instantiate ``Table`` or ``seq`` with. @@ -317,21 +319,20 @@ representObject .. code-block:: nim proc representObject*(value: MyObject, ts: TagStyle = tsNone, - c: SerializationContext): RawYamlStream {.raises: [].} + c: SerializationContext, tag: TagId): {.raises: [].} -This proc should return an iterator over ``YamlStreamEvent`` which represents -the type. Follow the following guidelines when implementing a custom -``representObject`` proc: +This proc should push a list of tokens that represent the type into the +serialization context via ``c.put``. Follow the following guidelines when +implementing a custom ``representObject`` proc: - You can use the helper template `presentTag `_ for outputting the tag. -- Always output the first tag with a ``yAnchorNone``. Anchors will be set +- Always output the first token with a ``yAnchorNone``. Anchors will be set automatically by ``ref`` type handling. - When outputting non-scalar types, you should use the ``representObject`` implementation of the child types, if possible. -- Check if the given ``TagStyle`` equals ``tsRootOnly`` and if yes, change it - to ``tsNone`` for the child values. +- Always use the ``tag`` parameter as tag for the first token you generate. - Never write a ``representObject`` proc for ``ref`` types. The following example for representing to a YAML scalar is the actual @@ -339,28 +340,21 @@ implementation of representing ``int`` types: .. code-block:: nim - proc representObject*[T: uint8|uint16|uint32|uint64]( - value: T, ts: TagStyle, c: SerializationContext): - RawYamlStream {.raises: [].} = - result = iterator(): YamlStreamEvent = - yield scalarEvent($value, presentTag(T, ts), yAnchorNone) + proc representObject*[T: int8|int16|int32|int64](value: T, ts: TagStyle, + c: SerializationContext, tag: TagId) {.raises: [].} = + ## represents an integer value as YAML scalar + c.put(scalarEvent($value, tag, yAnchorNone)) The following example for representing to a YAML non-scalar is the actual -implementation of representing ``seq`` types: +implementation of representing ``seq`` and ``set`` types: .. code-block:: nim - proc representObject*[T](value: seq[T], ts: TagStyle, - c: SerializationContext): RawYamlStream {.raises: [].} = - result = iterator(): YamlStreamEvent = - let childTagStyle = if ts == tsRootOnly: tsNone else: ts - yield YamlStreamEvent(kind: yamlStartSequence, - seqTag: presentTag(seq[T], ts), - seqAnchor: yAnchorNone) - for item in value: - var events = representObject(item, childTagStyle, c) - while true: - let event = events() - if finished(events): break - yield event - yield YamlStreamEvent(kind: yamlEndSequence) \ No newline at end of file + proc representObject*[T](value: seq[T]|set[T], ts: TagStyle, + c: SerializationContext, tag: TagId) {.raises: [YamlStreamError].} = + ## represents a Nim seq as YAML sequence + let childTagStyle = if ts == tsRootOnly: tsNone else: ts + c.put(startSeqEvent(tag)) + for item in value: + representChild(item, childTagStyle, c) + c.put(endSeqEvent()) \ No newline at end of file diff --git a/doc/style.css b/doc/style.css index 2b93efb..3be9e52 100644 --- a/doc/style.css +++ b/doc/style.css @@ -18,6 +18,11 @@ header a { padding-right: 5px; } +header a.active { + background: #877 !important; + color: black !important; +} + header span { display: inline-block; line-height: 50px; @@ -27,6 +32,34 @@ header span { padding-right: 5px; } +header span a { + display: block; +} + +header span ul { + display: none; + position: absolute; + top: 100%; + list-style: none; + background: #111; + margin: 0; +} + +header span ul:after { + content: ""; clear: both; display: block; +} + +header span:hover > ul { + display: block; +} + +header span ul a { + font-size: smaller; + font-family: "Source Code Pro", Menlo, "Courier New", Courier, monospace; + padding: 0 10px; + line-height: 40px; +} + header a:link, header a:visited { background: inherit; diff --git a/nimdoc.cfg b/nimdoc.cfg index acc34d7..fcea78a 100644 --- a/nimdoc.cfg +++ b/nimdoc.cfg @@ -98,7 +98,7 @@ doc.file = """ - + @@ -111,7 +111,20 @@ doc.file = """ Docs: Overview Serialization - Module yaml + + Modules + +
diff --git a/private/dom.nim b/private/dom.nim deleted file mode 100644 index 1b4093e..0000000 --- a/private/dom.nim +++ /dev/null @@ -1,195 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -proc newYamlNode*(content: string, tag: string = "?"): YamlNode = - YamlNode(kind: yScalar, content: content, tag: tag) - -proc newYamlNode*(children: openarray[YamlNode], tag: string = "?"): - YamlNode = - YamlNode(kind: ySequence, children: @children, tag: tag) - -proc newYamlNode*(pairs: openarray[tuple[key, value: YamlNode]], - tag: string = "?"): YamlNode = - YamlNode(kind: yMapping, pairs: @pairs, tag: tag) - -proc initYamlDoc*(root: YamlNode): YamlDocument = result.root = root - -proc composeNode(s: var YamlStream, tagLib: TagLibrary, - c: ConstructionContext): - YamlNode {.raises: [YamlStreamError, YamlConstructionError].} = - let start = s.next() - new(result) - try: - case start.kind - of yamlStartMap: - result.tag = tagLib.uri(start.mapTag) - result.kind = yMapping - result.pairs = newSeq[tuple[key, value: YamlNode]]() - while s.peek().kind != yamlEndMap: - let - key = composeNode(s, tagLib, c) - value = composeNode(s, tagLib, c) - result.pairs.add((key: key, value: value)) - discard s.next() - if start.mapAnchor != yAnchorNone: - assert(not c.refs.hasKey(start.mapAnchor)) - c.refs[start.mapAnchor] = cast[pointer](result) - of yamlStartSeq: - result.tag = tagLib.uri(start.seqTag) - result.kind = ySequence - result.children = newSeq[YamlNode]() - while s.peek().kind != yamlEndSeq: - result.children.add(composeNode(s, tagLib, c)) - if start.seqAnchor != yAnchorNone: - assert(not c.refs.hasKey(start.seqAnchor)) - c.refs[start.seqAnchor] = cast[pointer](result) - discard s.next() - of yamlScalar: - result.tag = tagLib.uri(start.scalarTag) - result.kind = yScalar - shallowCopy(result.content, start.scalarContent) - if start.scalarAnchor != yAnchorNone: - assert(not c.refs.hasKey(start.scalarAnchor)) - c.refs[start.scalarAnchor] = cast[pointer](result) - of yamlAlias: - result = cast[YamlNode](c.refs[start.aliasTarget]) - else: assert false, "Malformed YamlStream" - except KeyError: - raise newException(YamlConstructionError, - "Wrong tag library: TagId missing") - -proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument - {.raises: [YamlStreamError, YamlConstructionError].} = - var context = newConstructionContext() - assert s.next().kind == yamlStartDoc, "Malformed YamlStream" - result.root = composeNode(s, tagLib, context) - assert s.next().kind == yamlEndDoc, "Malformed YamlStream" - -proc loadDOM*(s: Stream): YamlDocument - {.raises: [IOError, YamlParserError, YamlConstructionError].} = - var - tagLib = initExtendedTagLibrary() - parser = newYamlParser(tagLib) - events = parser.parse(s) - try: result = compose(events, tagLib) - except YamlStreamError: - let e = getCurrentException() - if e.parent of YamlParserError: - raise (ref YamlParserError)(e.parent) - elif e.parent of IOError: - raise (ref IOError)(e.parent) - else: assert false, "Never happens: " & e.parent.repr - -proc serializeNode(n: YamlNode, c: SerializationContext, a: AnchorStyle, - tagLib: TagLibrary): RawYamlStream {.raises: [].}= - let p = cast[pointer](n) - if a != asNone and c.refs.hasKey(p): - try: - if c.refs[p] == yAnchorNone: - c.refs[p] = c.nextAnchorId - c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1) - except KeyError: assert false, "Can never happen" - result = iterator(): YamlStreamEvent {.raises: [].} = - var event: YamlStreamEvent - try: event = aliasEvent(c.refs[p]) - except KeyError: assert false, "Can never happen" - yield event - return - var - tagId: TagId - anchor: AnchorId - try: - if a == asAlways: - c.refs[p] = c.nextAnchorId - c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1) - else: c.refs[p] = yAnchorNone - tagId = if tagLib.tags.hasKey(n.tag): tagLib.tags[n.tag] else: - tagLib.registerUri(n.tag) - case a - of asNone: anchor = yAnchorNone - of asTidy: anchor = cast[AnchorId](n) - of asAlways: anchor = c.refs[p] - except KeyError: assert false, "Can never happen" - result = iterator(): YamlStreamEvent = - case n.kind - of yScalar: yield scalarEvent(n.content, tagId, anchor) - of ySequence: - yield startSeqEvent(tagId, anchor) - for item in n.children: - var events = serializeNode(item, c, a, tagLib) - while true: - let event = events() - if finished(events): break - yield event - yield endSeqEvent() - of yMapping: - yield startMapEvent(tagId, anchor) - for i in n.pairs: - var events = serializeNode(i.key, c, a, tagLib) - while true: - let event = events() - if finished(events): break - yield event - events = serializeNode(i.value, c, a, tagLib) - while true: - let event = events() - if finished(events): break - yield event - yield endMapEvent() - -template processAnchoredEvent(target: expr, c: SerializationContext): stmt = - try: - let anchorId = c.refs[cast[pointer](target)] - if anchorId != yAnchorNone: target = anchorId - else: target = yAnchorNone - except KeyError: assert false, "Can never happen" - yield event - -proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy): - YamlStream {.raises: [].} = - var - context = newSerializationContext(a) - events = serializeNode(doc.root, context, a, tagLib) - if a == asTidy: - var backend = iterator(): YamlStreamEvent {.raises: [].} = - var output = newSeq[YamlStreamEvent]() - while true: - let event = events() - if finished(events): break - output.add(event) - yield startDocEvent() - for event in output.mitems(): - case event.kind - of yamlScalar: processAnchoredEvent(event.scalarAnchor, context) - of yamlStartMap: processAnchoredEvent(event.mapAnchor, context) - of yamlStartSeq: processAnchoredEvent(event.seqAnchor, context) - else: yield event - yield endDocEvent() - result = initYamlStream(backend) - else: - var backend = iterator(): YamlStreamEvent {.raises: [].} = - yield startDocEvent() - while true: - let event = events() - if finished(events): break - yield event - yield endDocEvent() - result = initYamlStream(backend) - -proc dumpDOM*(doc: YamlDocument, target: Stream, - anchorStyle: AnchorStyle = asTidy, - options: PresentationOptions = defaultPresentationOptions) - {.raises: [YamlPresenterJsonError, YamlPresenterOutputError].} = - ## Dump a YamlDocument as YAML character stream. - var - tagLib = initExtendedTagLibrary() - events = serialize(doc, tagLib, - if options.style == psJson: asNone else: anchorStyle) - try: - present(events, target, tagLib, options) - except YamlStreamError: - # serializing object does not raise any errors, so we can ignore this - assert false, "Can never happen" \ No newline at end of file diff --git a/private/events.nim b/private/events.nim deleted file mode 100644 index 395b15a..0000000 --- a/private/events.nim +++ /dev/null @@ -1,73 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool = - if left.kind != right.kind: return false - case left.kind - of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlEndSeq: result = true - of yamlStartMap: - result = left.mapAnchor == right.mapAnchor and left.mapTag == right.mapTag - of yamlStartSeq: - result = left.seqAnchor == right.seqAnchor and left.seqTag == right.seqTag - of yamlScalar: - result = left.scalarAnchor == right.scalarAnchor and - left.scalarTag == right.scalarTag and - left.scalarContent == right.scalarContent - of yamlAlias: result = left.aliasTarget == right.aliasTarget - -proc `$`*(event: YamlStreamEvent): string = - result = $event.kind & '(' - case event.kind - of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard - of yamlStartMap: - result &= "tag=" & $event.mapTag - if event.mapAnchor != yAnchorNone: result &= ", anchor=" & $event.mapAnchor - of yamlStartSeq: - result &= "tag=" & $event.seqTag - if event.seqAnchor != yAnchorNone: result &= ", anchor=" & $event.seqAnchor - of yamlScalar: - result &= "tag=" & $event.scalarTag - if event.scalarAnchor != yAnchorNone: - result &= ", anchor=" & $event.scalarAnchor - result &= ", content=\"" & event.scalarContent & '\"' - of yamlAlias: - result &= "aliasTarget=" & $event.aliasTarget - result &= ")" - -proc tag*(event: YamlStreamEvent): TagId = - case event.kind - of yamlStartMap: result = event.mapTag - of yamlStartSeq: result = event.seqTag - of yamlScalar: result = event.scalarTag - else: raise newException(FieldError, "Event " & $event.kind & " has no tag") - -proc startDocEvent*(): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlStartDoc) - -proc endDocEvent*(): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlEndDoc) - -proc startMapEvent*(tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlStartMap, mapTag: tag, mapAnchor: anchor) - -proc endMapEvent*(): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlEndMap) - -proc startSeqEvent*(tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlStartSeq, seqTag: tag, seqAnchor: anchor) - -proc endSeqEvent*(): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlEndSeq) - -proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag, - scalarAnchor: anchor, scalarContent: content) - -proc aliasEvent*(anchor: AnchorId): YamlStreamEvent = - result = YamlStreamEvent(kind: yamlAlias, aliasTarget: anchor) \ No newline at end of file diff --git a/private/fastparse.nim b/private/fastparse.nim deleted file mode 100644 index 12dd460..0000000 --- a/private/fastparse.nim +++ /dev/null @@ -1,1551 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -type - FastParseState = enum - fpInitial, fpBlockAfterObject, fpBlockAfterPlainScalar, fpBlockObjectStart, - fpExpectDocEnd, fpFlow, fpFlowAfterObject, fpAfterDocument - - ScalarType = enum - stFlow, stLiteral, stFolded - - LexedDirective = enum - ldYaml, ldTag, ldUnknown - - YamlContext = enum - cBlock, cFlow - - ChompType = enum - ctKeep, ctClip, ctStrip - -const - space = {' ', '\t'} - lineEnd = {'\l', '\c', EndOfFile} - spaceOrLineEnd = {' ', '\t', '\l', '\c', EndOfFile} - digits = {'0'..'9'} - flowIndicators = {'[', ']', '{', '}', ','} - - UTF8NextLine = toUTF8(0x85.Rune) - UTF8NonBreakingSpace = toUTF8(0xA0.Rune) - UTF8LineSeparator = toUTF8(0x2028.Rune) - UTF8ParagraphSeparator = toUTF8(0x2029.Rune) - UnknownIndentation = int.low - -proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(), - callback: WarningCallback = nil): YamlParser = - new(result) - result.tagLib = tagLib - result.callback = callback - result.content = "" - result.after = "" - result.tagUri = "" - result.ancestry = newSeq[FastParseLevel]() - -proc getLineNumber*(p: YamlParser): int = p.lexer.lineNumber - -proc getColNumber*(p: YamlParser): int = p.tokenstart + 1 # column is 1-based - -proc getLineContent*(p: YamlParser, marker: bool = true): string = - result = p.lexer.getCurrentLine(false) - if marker: result.add(repeat(' ', p.tokenstart) & "^\n") - -template debug(message: string) {.dirty.} = - when defined(yamlDebug): - try: styledWriteLine(stdout, fgBlue, message) - except IOError: discard - -template debugFail() {.dirty.} = - when not defined(release): - echo "internal error at line: ", instantiationInfo().line - assert(false) - -proc generateError(p: YamlParser, message: string): - ref YamlParserError {.raises: [].} = - result = newException(YamlParserError, message) - result.line = p.lexer.lineNumber - result.column = p.tokenstart + 1 - result.lineContent = p.getLineContent(true) - -proc generateError(lx: BaseLexer, message: string): - ref YamlParserError {.raises: [].} = - result = newException(YamlParserError, message) - result.line = lx.lineNumber - result.column = lx.bufpos + 1 - result.lineContent = lx.getCurrentLine(false) & - repeat(' ', lx.getColNumber(lx.bufpos)) & "^\n" - -proc addMultiple(s: var string, c: char, num: int) {.raises: [], inline.} = - for i in 1..num: - s.add(c) - -proc reset(buffer: var string) {.raises: [], inline.} = buffer.setLen(0) - -proc initLevel(k: FastParseLevelKind): FastParseLevel {.raises: [], inline.} = - FastParseLevel(kind: k, indentation: UnknownIndentation) - -proc emptyScalar(p: YamlParser): YamlStreamEvent {.raises: [], inline.} = - result = scalarEvent("", p.tag, p.anchor) - p.tag = yTagQuestionMark - p.anchor = yAnchorNone - -proc currentScalar(p: YamlParser): YamlStreamEvent {.raises: [], inline.} = - result = YamlStreamEvent(kind: yamlScalar, scalarTag: p.tag, - scalarAnchor: p.anchor, scalarContent: p.content) - p.tag = yTagQuestionMark - p.anchor = yAnchorNone - -template yieldLevelEnd() {.dirty.} = - case p.level.kind - of fplSequence: yield endSeqEvent() - of fplMapKey: yield endMapEvent() - of fplMapValue, fplSinglePairValue: - yield emptyScalar(p) - yield endMapEvent() - of fplScalar: - if scalarType != stFlow: - case chomp - of ctKeep: - if p.content.len == 0: p.newlines.inc(-1) - p.content.addMultiple('\l', p.newlines) - of ctClip: - if p.content.len != 0: p.content.add('\l') - of ctStrip: discard - yield currentScalar(p) - p.tag = yTagQuestionMark - p.anchor = yAnchorNone - of fplUnknown: - if p.ancestry.len > 1: - yield emptyScalar(p) # don't yield scalar for empty doc - of fplSinglePairKey, fplDocument: debugFail() - -template handleLineEnd(insideDocument: bool) {.dirty.} = - case p.lexer.buf[p.lexer.bufpos] - of '\l': p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - of '\c': p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - of EndOfFile: - when insideDocument: closeEverything() - return - else: discard - p.newlines.inc() - -template handleObjectEnd(nextState: FastParseState) {.dirty.} = - p.level = p.ancestry.pop() - if p.level.kind == fplSinglePairValue: - yield endMapEvent() - p.level = p.ancestry.pop() - state = if p.level.kind == fplDocument: fpExpectDocEnd else: nextState - case p.level.kind - of fplMapKey: p.level.kind = fplMapValue - of fplSinglePairKey: p.level.kind = fplSinglePairValue - of fplMapValue: p.level.kind = fplMapKey - of fplSequence, fplDocument: discard - of fplUnknown, fplScalar, fplSinglePairValue: debugFail() - -proc objectStart(p: YamlParser, k: static[YamlStreamEventKind], - single: bool = false): YamlStreamEvent {.raises: [].} = - assert(p.level.kind == fplUnknown) - when k == yamlStartMap: - result = startMapEvent(p.tag, p.anchor) - if single: - debug("started single-pair map at " & - (if p.level.indentation == UnknownIndentation: $p.indentation else: - $p.level.indentation)) - p.level.kind = fplSinglePairKey - else: - debug("started map at " & - (if p.level.indentation == UnknownIndentation: $p.indentation else: - $p.level.indentation)) - p.level.kind = fplMapKey - else: - result = startSeqEvent(p.tag, p.anchor) - debug("started sequence at " & - (if p.level.indentation == UnknownIndentation: $p.indentation else: - $p.level.indentation)) - p.level.kind = fplSequence - p.tag = yTagQuestionMark - p.anchor = yAnchorNone - if p.level.indentation == UnknownIndentation: - p.level.indentation = p.indentation - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - -template closeMoreIndentedLevels(atSequenceItem: bool = false) {.dirty.} = - while p.level.kind != fplDocument: - let parent = p.ancestry[p.ancestry.high] - if parent.indentation >= p.indentation: - when atSequenceItem: - if (p.indentation == p.level.indentation and - p.level.kind == fplSequence) or - (p.indentation == parent.indentation and - p.level.kind == fplUnknown and parent.kind != fplSequence): - break - debug("Closing because parent.indentation (" & $parent.indentation & - ") >= indentation(" & $p.indentation & ")") - yieldLevelEnd() - handleObjectEnd(state) - else: break - if p.level.kind == fplDocument: state = fpExpectDocEnd - -template closeEverything() {.dirty.} = - p.indentation = 0 - closeMoreIndentedLevels() - case p.level.kind - of fplUnknown: discard p.ancestry.pop() - of fplDocument: discard - else: - yieldLevelEnd() - discard p.ancestry.pop() - yield endDocEvent() - -template handleBlockSequenceIndicator() {.dirty.} = - p.startToken() - case p.level.kind - of fplUnknown: yield p.objectStart(yamlStartSeq) - of fplSequence: - if p.level.indentation != p.indentation: - raise p.generateError("Invalid p.indentation of block sequence indicator") - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - else: raise p.generateError("Illegal sequence item in map") - p.lexer.skipWhitespace() - p.indentation = p.lexer.getColNumber(p.lexer.bufpos) - -template handleMapKeyIndicator() {.dirty.} = - p.startToken() - case p.level.kind - of fplUnknown: yield p.objectStart(yamlStartMap) - of fplMapValue: - if p.level.indentation != p.indentation: - raise p.generateError("Invalid p.indentation of map key indicator") - yield scalarEvent("", yTagQuestionMark, yAnchorNone) - p.level.kind = fplMapKey - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - of fplMapKey: - if p.level.indentation != p.indentation: - raise p.generateError("Invalid p.indentation of map key indicator") - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - of fplSequence: - raise p.generateError("Unexpected map key indicator (expected '- ')") - of fplScalar: - raise p.generateError( - "Unexpected map key indicator (expected multiline scalar end)") - of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail() - p.lexer.skipWhitespace() - p.indentation = p.lexer.getColNumber(p.lexer.bufpos) - -template handleMapValueIndicator() {.dirty.} = - p.startToken() - case p.level.kind - of fplUnknown: - if p.level.indentation == UnknownIndentation: - yield p.objectStart(yamlStartMap) - yield scalarEvent("", yTagQuestionMark, yAnchorNone) - else: yield emptyScalar(p) - p.ancestry[p.ancestry.high].kind = fplMapValue - of fplMapKey: - if p.level.indentation != p.indentation: - raise p.generateError("Invalid p.indentation of map key indicator") - yield scalarEvent("", yTagQuestionMark, yAnchorNone) - p.level.kind = fplMapValue - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - of fplMapValue: - if p.level.indentation != p.indentation: - raise p.generateError("Invalid p.indentation of map key indicator") - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - of fplSequence: - raise p.generateError("Unexpected map value indicator (expected '- ')") - of fplScalar: - raise p.generateError( - "Unexpected map value indicator (expected multiline scalar end)") - of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail() - p.lexer.skipWhitespace() - p.indentation = p.lexer.getColNumber(p.lexer.bufpos) - -proc initDocValues(p: YamlParser) {.raises: [].} = - p.shorthands = initTable[string, string]() - p.anchors = initTable[string, AnchorId]() - p.shorthands["!"] = "!" - p.shorthands["!!"] = "tag:yaml.org,2002:" - p.nextAnchorId = 0.AnchorId - p.level = initLevel(fplUnknown) - p.tag = yTagQuestionMark - p.anchor = yAnchorNone - p.ancestry.add(FastParseLevel(kind: fplDocument, indentation: -1)) - -proc startToken(p: YamlParser) {.raises: [], inline.} = - p.tokenstart = p.lexer.getColNumber(p.lexer.bufpos) - -proc anchorName(p: YamlParser) {.raises: [].} = - debug("lex: anchorName") - while true: - p.lexer.bufpos.inc() - let c = p.lexer.buf[p.lexer.bufpos] - case c - of spaceOrLineEnd, '[', ']', '{', '}', ',': break - else: p.content.add(c) - -proc handleAnchor(p: YamlParser) {.raises: [YamlParserError].} = - p.startToken() - if p.level.kind != fplUnknown: raise p.generateError("Unexpected token") - if p.anchor != yAnchorNone: - raise p.generateError("Only one anchor is allowed per node") - p.content.reset() - p.anchorName() - p.anchor = p.nextAnchorId - p.anchors[p.content] = p.anchor - p.nextAnchorId = AnchorId(int(p.nextAnchorId) + 1) - -template handleAlias() {.dirty.} = - p.startToken() - if p.level.kind != fplUnknown: raise p.generateError("Unexpected token") - if p.anchor != yAnchorNone or p.tag != yTagQuestionMark: - raise p.generateError("Alias may not have anchor or tag") - p.content.reset() - p.anchorName() - var id: AnchorId - try: id = p.anchors[p.content] - except KeyError: raise p.generateError("Unknown anchor") - yield aliasEvent(id) - handleObjectEnd(fpBlockAfterObject) - -template leaveFlowLevel() {.dirty.} = - flowdepth.inc(-1) - if flowdepth == 0: - yieldLevelEnd() - handleObjectEnd(fpBlockAfterObject) - else: - yieldLevelEnd() - handleObjectEnd(fpFlowAfterObject) - -template handlePossibleMapStart(flow: bool = false, - single: bool = false) {.dirty.} = - if p.level.indentation == UnknownIndentation: - var flowDepth = 0 - var pos = p.lexer.bufpos - var recentJsonStyle = false - while pos < p.lexer.bufpos + 1024: - case p.lexer.buf[pos] - of ':': - if flowDepth == 0 and (p.lexer.buf[pos + 1] in spaceOrLineEnd or - recentJsonStyle): - yield p.objectStart(yamlStartMap, single) - break - of lineEnd: break - of '[', '{': flowDepth.inc() - of '}', ']': - flowDepth.inc(-1) - if flowDepth < 0: break - of '?', ',': - if flowDepth == 0: break - of '#': - if p.lexer.buf[pos - 1] in space: break - of '"': - pos.inc() - while p.lexer.buf[pos] notin {'"', EndOfFile, '\l', '\c'}: - if p.lexer.buf[pos] == '\\': pos.inc() - pos.inc() - if p.lexer.buf[pos] != '"': break - of '\'': - pos.inc() - while p.lexer.buf[pos] notin {'\'', '\l', '\c', EndOfFile}: - pos.inc() - of '&', '*', '!': - if pos == p.lexer.bufpos or p.lexer.buf[p.lexer.bufpos] in space: - pos.inc() - while p.lexer.buf[pos] notin spaceOrLineEnd: - pos.inc() - continue - else: discard - if flow and p.lexer.buf[pos] notin space: - recentJsonStyle = p.lexer.buf[pos] in {']', '}', '\'', '"'} - pos.inc() - if p.level.indentation == UnknownIndentation: - p.level.indentation = p.indentation - -template handleBlockItemStart() {.dirty.} = - case p.level.kind - of fplUnknown: handlePossibleMapStart() - of fplSequence: - raise p.generateError( - "Unexpected token (expected block sequence indicator)") - of fplMapKey: - p.ancestry.add(p.level) - p.level = FastParseLevel(kind: fplUnknown, indentation: p.indentation) - of fplMapValue: - yield emptyScalar(p) - p.level.kind = fplMapKey - p.ancestry.add(p.level) - p.level = FastParseLevel(kind: fplUnknown, indentation: p.indentation) - of fplScalar, fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail() - -template handleFlowItemStart() {.dirty.} = - if p.level.kind == fplUnknown and - p.ancestry[p.ancestry.high].kind == fplSequence: - handlePossibleMapStart(true, true) - -proc finishLine(lexer: var BaseLexer) {.raises: [], inline.} = - debug("lex: finishLine") - while lexer.buf[lexer.bufpos] notin lineEnd: - lexer.bufpos.inc() - -proc skipWhitespace(lexer: var BaseLexer) {.raises: [], inline.} = - debug("lex: skipWhitespace") - while lexer.buf[lexer.bufpos] in space: lexer.bufpos.inc() - -# TODO: {.raises: [].} -proc skipWhitespaceCommentsAndNewlines(lexer: var BaseLexer) {.inline.} = - debug("lex: skipWhitespaceCommentsAndNewlines") - if lexer.buf[lexer.bufpos] != '#': - while true: - case lexer.buf[lexer.bufpos] - of space: lexer.bufpos.inc() - of '\l': lexer.bufpos = lexer.handleLF(lexer.bufpos) - of '\c': lexer.bufpos = lexer.handleCR(lexer.bufpos) - of '#': # also skip comments - lexer.bufpos.inc() - while lexer.buf[lexer.bufpos] notin lineEnd: - lexer.bufpos.inc() - else: break - -proc skipIndentation(lexer: var BaseLexer) {.raises: [], inline.} = - debug("lex: skipIndentation") - while lexer.buf[lexer.bufpos] == ' ': lexer.bufpos.inc() - -proc directiveName(lexer: var BaseLexer, directive: var LexedDirective) - {.raises: [].} = - debug("lex: directiveName") - directive = ldUnknown - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] == 'Y': - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] == 'A': - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] == 'M': - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] == 'L': - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] in spaceOrLineEnd: - directive = ldYaml - elif lexer.buf[lexer.bufpos] == 'T': - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] == 'A': - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] == 'G': - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] in spaceOrLineEnd: - directive = ldTag - while lexer.buf[lexer.bufpos] notin spaceOrLineEnd: - lexer.bufpos.inc() - -proc yamlVersion(lexer: var BaseLexer, o: var string) - {.raises: [YamlParserError], inline.} = - debug("lex: yamlVersion") - while lexer.buf[lexer.bufpos] in space: lexer.bufpos.inc() - var c = lexer.buf[lexer.bufpos] - if c notin digits: raise lexer.generateError("Invalid YAML version number") - o.add(c) - lexer.bufpos.inc() - c = lexer.buf[lexer.bufpos] - while c in digits: - lexer.bufpos.inc() - o.add(c) - c = lexer.buf[lexer.bufpos] - if lexer.buf[lexer.bufpos] != '.': - raise lexer.generateError("Invalid YAML version number") - o.add('.') - lexer.bufpos.inc() - c = lexer.buf[lexer.bufpos] - if c notin digits: raise lexer.generateError("Invalid YAML version number") - o.add(c) - lexer.bufpos.inc() - c = lexer.buf[lexer.bufpos] - while c in digits: - o.add(c) - lexer.bufpos.inc() - c = lexer.buf[lexer.bufpos] - if lexer.buf[lexer.bufpos] notin spaceOrLineEnd: - raise lexer.generateError("Invalid YAML version number") - -proc lineEnding(p: YamlParser) {.raises: [YamlParserError], inline.} = - debug("lex: lineEnding") - if p.lexer.buf[p.lexer.bufpos] notin lineEnd: - while p.lexer.buf[p.lexer.bufpos] in space: p.lexer.bufpos.inc() - if p.lexer.buf[p.lexer.bufpos] in lineEnd: discard - elif p.lexer.buf[p.lexer.bufpos] == '#': - while p.lexer.buf[p.lexer.bufpos] notin lineEnd: p.lexer.bufpos.inc() - else: - p.startToken() - raise p.generateError("Unexpected token (expected comment or line end)") - -proc tagShorthand(lexer: var BaseLexer, shorthand: var string) {.inline.} = - debug("lex: tagShorthand") - while lexer.buf[lexer.bufpos] in space: lexer.bufpos.inc() - assert lexer.buf[lexer.bufpos] == '!' - shorthand.add('!') - lexer.bufpos.inc() - var c = lexer.buf[lexer.bufpos] - if c in spaceOrLineEnd: discard - else: - while c != '!': - case c - of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '-': - shorthand.add(c) - lexer.bufpos.inc() - c = lexer.buf[lexer.bufpos] - else: raise lexer.generateError("Illegal character in tag shorthand") - shorthand.add(c) - lexer.bufpos.inc() - if lexer.buf[lexer.bufpos] notin spaceOrLineEnd: - raise lexer.generateError("Missing space after tag shorthand") - -proc tagUriMapping(lexer: var BaseLexer, uri: var string) - {.raises: [YamlParserError].} = - debug("lex: tagUriMapping") - while lexer.buf[lexer.bufpos] in space: - lexer.bufpos.inc() - var c = lexer.buf[lexer.bufpos] - if c == '!': - uri.add(c) - lexer.bufpos.inc() - c = lexer.buf[lexer.bufpos] - while c notin spaceOrLineEnd: - case c - of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', '@', '&', - '-', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')': - uri.add(c) - lexer.bufpos.inc() - c = lexer.buf[lexer.bufpos] - else: raise lexer.generateError("Invalid tag uri") - -proc directivesEndMarker(lexer: var BaseLexer, success: var bool) - {.raises: [].} = - debug("lex: directivesEndMarker") - success = true - for i in 0..2: - if lexer.buf[lexer.bufpos + i] != '-': - success = false - break - if success: success = lexer.buf[lexer.bufpos + 3] in spaceOrLineEnd - -proc documentEndMarker(lexer: var BaseLexer, success: var bool) {.raises: [].} = - debug("lex: documentEndMarker") - success = true - for i in 0..2: - if lexer.buf[lexer.bufpos + i] != '.': - success = false - break - if success: success = lexer.buf[lexer.bufpos + 3] in spaceOrLineEnd - -proc unicodeSequence(lexer: var BaseLexer, length: int): - string {.raises: [YamlParserError].} = - debug("lex: unicodeSequence") - var unicodeChar = 0.int - for i in countup(0, length - 1): - lexer.bufpos.inc() - let - digitPosition = length - i - 1 - c = lexer.buf[lexer.bufpos] - case c - of EndOFFile, '\l', '\c': - raise lexer.generateError("Unfinished unicode escape sequence") - of '0' .. '9': - unicodeChar = unicodechar or (int(c) - 0x30) shl (digitPosition * 4) - of 'A' .. 'F': - unicodeChar = unicodechar or (int(c) - 0x37) shl (digitPosition * 4) - of 'a' .. 'f': - unicodeChar = unicodechar or (int(c) - 0x57) shl (digitPosition * 4) - else: - raise lexer.generateError( - "Invalid character in unicode escape sequence") - return toUTF8(Rune(unicodeChar)) - -proc byteSequence(lexer: var BaseLexer): char {.raises: [YamlParserError].} = - debug("lex: byteSequence") - var charCode = 0.int8 - for i in 0 .. 1: - lexer.bufpos.inc() - let - digitPosition = int8(1 - i) - c = lexer.buf[lexer.bufpos] - case c - of EndOfFile, '\l', 'r': - raise lexer.generateError("Unfinished octet escape sequence") - of '0' .. '9': - charCode = charCode or (int8(c) - 0x30.int8) shl (digitPosition * 4) - of 'A' .. 'F': - charCode = charCode or (int8(c) - 0x37.int8) shl (digitPosition * 4) - of 'a' .. 'f': - charCode = charCode or (int8(c) - 0x57.int8) shl (digitPosition * 4) - else: - raise lexer.generateError("Invalid character in octet escape sequence") - return char(charCode) - -# TODO: {.raises: [].} -proc processQuotedWhitespace(p: YamlParser, newlines: var int) = - p.after.reset() - block outer: - while true: - case p.lexer.buf[p.lexer.bufpos] - of ' ', '\t': p.after.add(p.lexer.buf[p.lexer.bufpos]) - of '\l': - p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - break - of '\c': - p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - break - else: - p.content.add(p.after) - break outer - p.lexer.bufpos.inc() - while true: - case p.lexer.buf[p.lexer.bufpos] - of ' ', '\t': discard - of '\l': - p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - newlines.inc() - continue - of '\c': - p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - newlines.inc() - continue - else: - if newlines == 0: discard - elif newlines == 1: p.content.add(' ') - else: p.content.addMultiple('\l', newlines - 1) - break - p.lexer.bufpos.inc() - -# TODO: {.raises: [YamlParserError].} -proc doubleQuotedScalar(p: YamlParser) = - debug("lex: doubleQuotedScalar") - p.lexer.bufpos.inc() - while true: - var c = p.lexer.buf[p.lexer.bufpos] - case c - of EndOfFile: - raise p.lexer.generateError("Unfinished double quoted string") - of '\\': - p.lexer.bufpos.inc() - case p.lexer.buf[p.lexer.bufpos] - of EndOfFile: - raise p.lexer.generateError("Unfinished escape sequence") - of '0': p.content.add('\0') - of 'a': p.content.add('\x07') - of 'b': p.content.add('\x08') - of '\t', 't': p.content.add('\t') - of 'n': p.content.add('\l') - of 'v': p.content.add('\v') - of 'f': p.content.add('\f') - of 'r': p.content.add('\c') - of 'e': p.content.add('\e') - of ' ': p.content.add(' ') - of '"': p.content.add('"') - of '/': p.content.add('/') - of '\\': p.content.add('\\') - of 'N': p.content.add(UTF8NextLine) - of '_': p.content.add(UTF8NonBreakingSpace) - of 'L': p.content.add(UTF8LineSeparator) - of 'P': p.content.add(UTF8ParagraphSeparator) - of 'x': p.content.add(p.lexer.unicodeSequence(2)) - of 'u': p.content.add(p.lexer.unicodeSequence(4)) - of 'U': p.content.add(p.lexer.unicodeSequence(8)) - of '\l', '\c': - var newlines = 0 - p.processQuotedWhitespace(newlines) - continue - else: raise p.lexer.generateError("Illegal character in escape sequence") - of '"': - p.lexer.bufpos.inc() - break - of '\l', '\c', '\t', ' ': - var newlines = 1 - p.processQuotedWhitespace(newlines) - continue - else: p.content.add(c) - p.lexer.bufpos.inc() - -# TODO: {.raises: [].} -proc singleQuotedScalar(p: YamlParser) = - debug("lex: singleQuotedScalar") - p.lexer.bufpos.inc() - while true: - case p.lexer.buf[p.lexer.bufpos] - of '\'': - p.lexer.bufpos.inc() - if p.lexer.buf[p.lexer.bufpos] == '\'': p.content.add('\'') - else: break - of EndOfFile: raise p.lexer.generateError("Unfinished single quoted string") - of '\l', '\c', '\t', ' ': - var newlines = 1 - p.processQuotedWhitespace(newlines) - continue - else: p.content.add(p.lexer.buf[p.lexer.bufpos]) - p.lexer.bufpos.inc() - -proc isPlainSafe(lexer: BaseLexer, index: int, context: YamlContext): bool - {.raises: [].} = - case lexer.buf[lexer.bufpos + 1] - of spaceOrLineEnd: result = false - of flowIndicators: result = context == cBlock - else: result = true - - -# tried this for performance optimization, but it didn't optimize any -# performance. keeping it around for future reference. -#const -# plainCharOut = {'!', '\"', '$'..'9', ';'..'\xFF'} -# plainCharIn = {'!', '\"', '$'..'+', '-'..'9', ';'..'Z', '\\', '^'..'z', -# '|', '~'..'\xFF'} -#template isPlainChar(c: char, context: YamlContext): bool = -# when context == cBlock: c in plainCharOut -# else: c in plainCharIn - -proc plainScalar(p: YamlParser, context: static[YamlContext]) {.raises: [].} = - debug("lex: plainScalar") - p.content.add(p.lexer.buf[p.lexer.bufpos]) - block outer: - while true: - p.lexer.bufpos.inc() - let c = p.lexer.buf[p.lexer.bufpos] - case c - of ' ', '\t': - p.after.setLen(1) - p.after[0] = c - while true: - p.lexer.bufpos.inc() - let c2 = p.lexer.buf[p.lexer.bufpos] - case c2 - of ' ', '\t': p.after.add(c2) - of lineEnd: break outer - of ':': - if p.lexer.isPlainSafe(p.lexer.bufpos + 1, context): - p.content.add(p.after & ':') - break - else: break outer - of '#': break outer - of flowIndicators: - if context == cBlock: - p.content.add(p.after) - p.content.add(c2) - break - else: break outer - else: - p.content.add(p.after) - p.content.add(c2) - break - of flowIndicators: - when context == cFlow: break - else: p.content.add(c) - of lineEnd: break - of ':': - if p.lexer.isPlainSafe(p.lexer.bufpos + 1, context): p.content.add(':') - else: break outer - else: p.content.add(c) - debug("lex: \"" & p.content & '\"') - -proc continueMultilineScalar(p: YamlParser) {.raises: [].} = - p.content.add(if p.newlines == 1: " " else: repeat('\l', p.newlines - 1)) - p.startToken() - p.plainScalar(cBlock) - -template handleFlowPlainScalar() {.dirty.} = - p.content.reset() - p.startToken() - p.plainScalar(cFlow) - if p.lexer.buf[p.lexer.bufpos] in {'{', '}', '[', ']', ',', ':', '#'}: - discard - else: - p.newlines = 0 - while true: - case p.lexer.buf[p.lexer.bufpos] - of ':': - if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cFlow): - if p.newlines == 1: - p.content.add(' ') - p.newlines = 0 - elif p.newlines > 1: - p.content.addMultiple(' ', p.newlines - 1) - p.newlines = 0 - p.plainScalar(cFlow) - break - of '#', EndOfFile: break - of '\l': - p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - p.newlines.inc() - of '\c': - p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - p.newlines.inc() - of flowIndicators: break - of ' ', '\t': p.lexer.skipWhitespace() - else: - if p.newlines == 1: - p.content.add(' ') - p.newlines = 0 - elif p.newlines > 1: - p.content.addMultiple(' ', p.newlines - 1) - p.newlines = 0 - p.plainScalar(cFlow) - yield currentScalar(p) - handleObjectEnd(fpFlowAfterObject) - -proc tagHandle(p: YamlParser, shorthandEnd: var int) - {.raises: [YamlParserError].} = - debug("lex: tagHandle") - shorthandEnd = 0 - p.content.add(p.lexer.buf[p.lexer.bufpos]) - var i = 0 - while true: - p.lexer.bufpos.inc() - i.inc() - let c = p.lexer.buf[p.lexer.bufpos] - case c - of spaceOrLineEnd: - if shorthandEnd == -1: - raise p.lexer.generateError("Unclosed verbatim tag") - break - of '!': - if shorthandEnd == -1 and i == 2: - p.content.add(c) - continue - elif shorthandEnd != 0: - raise p.lexer.generateError("Illegal character in tag suffix") - shorthandEnd = i - p.content.add(c) - of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', '@', '&', - '-', '=', '+', '$', '_', '.', '~', '*', '\'', '(', ')': - p.content.add(c) - of ',': - if shortHandEnd > 0: break # ',' after shorthand is flow indicator - p.content.add(c) - of '<': - if i == 1: - shorthandEnd = -1 - p.content.reset() - else: raise p.lexer.generateError("Illegal character in tag handle") - of '>': - if shorthandEnd == -1: - p.lexer.bufpos.inc() - if p.lexer.buf[p.lexer.bufpos] notin spaceOrLineEnd: - raise p.lexer.generateError("Missing space after verbatim tag handle") - break - else: raise p.lexer.generateError("Illegal character in tag handle") - of '%': - if shorthandEnd != 0: p.content.add(p.lexer.byteSequence()) - else: raise p.lexer.generateError("Illegal character in tag handle") - else: raise p.lexer.generateError("Illegal character in tag handle") - -proc handleTagHandle(p: YamlParser) {.raises: [YamlParserError].} = - p.startToken() - if p.level.kind != fplUnknown: raise p.generateError("Unexpected tag handle") - if p.tag != yTagQuestionMark: - raise p.generateError("Only one tag handle is allowed per node") - p.content.reset() - var - shorthandEnd: int - p.tagHandle(shorthandEnd) - if shorthandEnd != -1: - try: - p.tagUri.reset() - p.tagUri.add(p.shorthands[p.content[0..shorthandEnd]]) - p.tagUri.add(p.content[shorthandEnd + 1 .. ^1]) - except KeyError: - raise p.generateError( - "Undefined tag shorthand: " & p.content[0..shorthandEnd]) - try: p.tag = p.tagLib.tags[p.tagUri] - except KeyError: p.tag = p.tagLib.registerUri(p.tagUri) - else: - try: p.tag = p.tagLib.tags[p.content] - except KeyError: p.tag = p.tagLib.registerUri(p.content) - -proc consumeLineIfEmpty(p: YamlParser, newlines: var int): bool = - result = true - while true: - p.lexer.bufpos.inc() - case p.lexer.buf[p.lexer.bufpos] - of ' ', '\t': discard - of '\l': - p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - break - of '\c': - p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - break - of '#', EndOfFile: - p.lineEnding() - handleLineEnd(false) - break - else: - result = false - break - -template startScalar(t: ScalarType) {.dirty.} = - p.newlines = 0 - p.level.kind = fplScalar - scalarType = t - -template blockScalarHeader() {.dirty.} = - debug("lex: blockScalarHeader") - chomp = ctClip - p.level.indentation = UnknownIndentation - if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark - let t = if p.lexer.buf[p.lexer.bufpos] == '|': stLiteral else: stFolded - while true: - p.lexer.bufpos.inc() - case p.lexer.buf[p.lexer.bufpos] - of '+': - if chomp != ctClip: - raise p.lexer.generateError("Only one chomping indicator is allowed") - chomp = ctKeep - of '-': - if chomp != ctClip: - raise p.lexer.generateError("Only one chomping indicator is allowed") - chomp = ctStrip - of '1'..'9': - if p.level.indentation != UnknownIndentation: - raise p.lexer.generateError("Only one p.indentation indicator is allowed") - p.level.indentation = p.ancestry[p.ancestry.high].indentation + - ord(p.lexer.buf[p.lexer.bufpos]) - ord('\x30') - of spaceOrLineEnd: break - else: - raise p.lexer.generateError( - "Illegal character in block scalar header: '" & - p.lexer.buf[p.lexer.bufpos] & "'") - recentWasMoreIndented = false - p.lineEnding() - handleLineEnd(true) - startScalar(t) - p.content.reset() - -template blockScalarLine() {.dirty.} = - debug("lex: blockScalarLine") - if p.indentation < p.level.indentation: - if p.lexer.buf[p.lexer.bufpos] == '#': - # skip all following comment lines - while p.indentation > p.ancestry[p.ancestry.high].indentation: - p.lineEnding() - handleLineEnd(true) - p.newlines.inc(-1) - p.lexer.skipIndentation() - p.indentation = p.lexer.getColNumber(p.lexer.bufpos) - if p.indentation > p.ancestry[p.ancestry.high].indentation: - raise p.lexer.generateError( - "Invalid content in block scalar after comments") - closeMoreIndentedLevels() - else: - raise p.lexer.generateError( - "Invalid p.indentation (expected p.indentation of at least " & - $p.level.indentation & " spaces)") - else: - if p.level.indentation == UnknownIndentation: - if p.lexer.buf[p.lexer.bufpos] in lineEnd: - handleLineEnd(true) - continue - else: - p.level.indentation = p.indentation - p.content.addMultiple('\l', p.newlines) - elif p.indentation > p.level.indentation or - p.lexer.buf[p.lexer.bufpos] == '\t': - p.content.addMultiple('\l', p.newlines) - recentWasMoreIndented = true - p.content.addMultiple(' ', p.indentation - p.level.indentation) - elif scalarType == stFolded: - if recentWasMoreIndented: - recentWasMoreIndented = false - p.newlines.inc() - if p.newlines == 0: discard - elif p.newlines == 1: p.content.add(' ') - else: p.content.addMultiple('\l', p.newlines - 1) - else: p.content.addMultiple('\l', p.newlines) - p.newlines = 0 - while p.lexer.buf[p.lexer.bufpos] notin lineEnd: - p.content.add(p.lexer.buf[p.lexer.bufpos]) - p.lexer.bufpos.inc() - handleLineEnd(true) - -proc parse*(p: YamlParser, s: Stream): YamlStream = - p.content.reset() - p.after.reset() - p.tagUri.reset() - p.ancestry.setLen(0) - var backend = iterator(): YamlStreamEvent = - var - state = fpInitial - flowdepth: int = 0 - explicitFlowKey: bool - scalarType: ScalarType - recentWasMoreIndented: bool - chomp: ChompType - - p.lexer.open(s) - p.initDocValues() - - while true: - case state - of fpInitial: - debug("state: initial") - case p.lexer.buf[p.lexer.bufpos] - of '%': - var ld: LexedDirective - p.startToken() - p.lexer.directiveName(ld) - case ld - of ldYaml: - var version = "" - p.startToken() - p.lexer.yamlVersion(version) - if version != "1.2": - if p.callback != nil: - p.callback(p.lexer.lineNumber, p.getColNumber(), - p.getLineContent(), - "Version is not 1.2, but " & version) - discard - p.lineEnding() - handleLineEnd(false) - of ldTag: - var shorthand = "" - p.tagUri.reset() - p.startToken() - p.lexer.tagShorthand(shorthand) - p.lexer.tagUriMapping(p.tagUri) - p.shorthands[shorthand] = p.tagUri - p.lineEnding() - handleLineEnd(false) - of ldUnknown: - if p.callback != nil: - p.callback(p.lexer.lineNumber, p.getColNumber(), - p.getLineContent(), "Unknown directive") - p.lexer.finishLine() - handleLineEnd(false) - of ' ', '\t': - if not p.consumeLineIfEmpty(p.newlines): - p.indentation = p.lexer.getColNumber(p.lexer.bufpos) - yield startDocEvent() - state = fpBlockObjectStart - of '\l': p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - of '\c': p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - of EndOfFile: return - of '#': - p.lineEnding() - handleLineEnd(false) - of '-': - var success: bool - p.startToken() - p.lexer.directivesEndMarker(success) - yield startDocEvent() - if success: - p.lexer.bufpos.inc(3) - state = fpBlockObjectStart - else: - yield startDocEvent() - state = fpBlockObjectStart - of fpBlockAfterPlainScalar: - debug("state: blockAfterPlainScalar") - p.lexer.skipWhitespace() - case p.lexer.buf[p.lexer.bufpos] - of '\l': - if p.level.kind notin {fplUnknown, fplScalar}: - p.startToken() - raise p.generateError("Unexpected scalar") - startScalar(stFlow) - p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - p.newlines.inc() - state = fpBlockObjectStart - of '\c': - if p.level.kind notin {fplUnknown, fplScalar}: - p.startToken() - raise p.generateError("Unexpected scalar") - startScalar(stFlow) - p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - p.newlines.inc() - state = fpBlockObjectStart - else: - yield currentScalar(p) - handleObjectEnd(fpBlockAfterObject) - of fpBlockAfterObject: - debug("state: blockAfterObject") - p.lexer.skipWhitespace() - case p.lexer.buf[p.lexer.bufpos] - of EndOfFile: - closeEverything() - break - of '\l': - state = fpBlockObjectStart - p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - of '\c': - state = fpBlockObjectStart - p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - of ':': - case p.level.kind - of fplUnknown: yield p.objectStart(yamlStartMap) - of fplMapKey: - yield scalarEvent("", yTagQuestionMark, yAnchorNone) - p.level.kind = fplMapValue - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - of fplMapValue: - p.level.kind = fplMapValue - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - of fplSequence: - p.startToken() - raise p.generateError("Illegal token (expected sequence item)") - of fplScalar: - p.startToken() - raise p.generateError( - "Multiline scalars may not be implicit map keys") - of fplSinglePairKey, fplSinglePairValue, fplDocument: debugFail() - p.lexer.bufpos.inc() - p.lexer.skipWhitespace() - p.indentation = p.lexer.getColNumber(p.lexer.bufpos) - state = fpBlockObjectStart - of '#': - p.lineEnding() - handleLineEnd(true) - state = fpBlockObjectStart - else: - p.startToken() - raise p.generateError( - "Illegal token (expected ':', comment or line end)") - of fpBlockObjectStart: - debug("state: blockObjectStart") - p.lexer.skipIndentation() - p.indentation = p.lexer.getColNumber(p.lexer.bufpos) - if p.indentation == 0: - var success: bool - p.lexer.directivesEndMarker(success) - if success: - p.lexer.bufpos.inc(3) - closeEverything() - p.initDocValues() - yield startDocEvent() - continue - p.lexer.documentEndMarker(success) - if success: - closeEverything() - p.lexer.bufpos.inc(3) - p.lineEnding() - handleLineEnd(false) - state = fpAfterDocument - continue - if p.indentation <= p.ancestry[p.ancestry.high].indentation: - if p.lexer.buf[p.lexer.bufpos] in lineEnd: - handleLineEnd(true) - continue - elif p.lexer.buf[p.lexer.bufpos] == '#': - p.lineEnding() - handleLineEnd(true) - continue - elif p.lexer.buf[p.lexer.bufpos] == '-' and not - p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock): - closeMoreIndentedLevels(true) - else: closeMoreIndentedLevels() - elif p.indentation <= p.level.indentation and - p.lexer.buf[p.lexer.bufpos] in lineEnd: - handleLineEnd(true) - continue - if p.level.kind == fplScalar and scalarType != stFlow: - blockScalarLine() - continue - case p.lexer.buf[p.lexer.bufpos] - of '\l': - p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - p.newlines.inc() - if p.level.kind == fplUnknown: - p.level.indentation = UnknownIndentation - of '\c': - p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - p.newlines.inc() - if p.level.kind == fplUnknown: - p.level.indentation = UnknownIndentation - of EndOfFile: - closeEverything() - return - of '#': - p.lineEnding() - handleLineEnd(true) - if p.level.kind == fplUnknown: - p.level.indentation = UnknownIndentation - of '\'': - handleBlockItemStart() - p.content.reset() - p.startToken() - p.singleQuotedScalar() - if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark - yield currentScalar(p) - handleObjectEnd(fpBlockAfterObject) - of '"': - handleBlockItemStart() - p.content.reset() - p.startToken() - p.doubleQuotedScalar() - if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark - yield currentScalar(p) - handleObjectEnd(fpBlockAfterObject) - of '|', '>': - blockScalarHeader() - continue - of '-': - if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock): - if p.level.kind == fplScalar: - p.continueMultilineScalar() - state = fpBlockAfterPlainScalar - else: - handleBlockItemStart() - p.content.reset() - p.startToken() - p.plainScalar(cBlock) - state = fpBlockAfterPlainScalar - else: - p.lexer.bufpos.inc() - handleBlockSequenceIndicator() - of '!': - handleBlockItemStart() - p.handleTagHandle() - of '&': - handleBlockItemStart() - p.handleAnchor() - of '*': - handleBlockItemStart() - handleAlias() - of '[', '{': - handleBlockItemStart() - state = fpFlow - of '?': - if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock): - if p.level.kind == fplScalar: - p.continueMultilineScalar() - state = fpBlockAfterPlainScalar - else: - handleBlockItemStart() - p.content.reset() - p.startToken() - p.plainScalar(cBlock) - state = fpBlockAfterPlainScalar - else: - p.lexer.bufpos.inc() - handleMapKeyIndicator() - of ':': - if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cBlock): - if p.level.kind == fplScalar: - p.continueMultilineScalar() - state = fpBlockAfterPlainScalar - else: - handleBlockItemStart() - p.content.reset() - p.startToken() - p.plainScalar(cBlock) - state = fpBlockAfterPlainScalar - else: - p.lexer.bufpos.inc() - handleMapValueIndicator() - of '@', '`': - raise p.lexer.generateError( - "Reserved characters cannot start a plain scalar") - of '\t': - if p.level.kind == fplScalar: - p.lexer.skipWhitespace() - p.continueMultilineScalar() - state = fpBlockAfterPlainScalar - else: raise p.lexer.generateError("\\t cannot start any token") - else: - if p.level.kind == fplScalar: - p.continueMultilineScalar() - state = fpBlockAfterPlainScalar - else: - handleBlockItemStart() - p.content.reset() - p.startToken() - p.plainScalar(cBlock) - state = fpBlockAfterPlainScalar - of fpExpectDocEnd: - debug("state: expectDocEnd") - case p.lexer.buf[p.lexer.bufpos] - of '-': - var success: bool - p.lexer.directivesEndMarker(success) - if success: - p.lexer.bufpos.inc(3) - yield endDocEvent() - discard p.ancestry.pop() - p.initDocValues() - yield startDocEvent() - state = fpBlockObjectStart - else: - raise p.generateError("Unexpected content (expected document end)") - of '.': - var isDocumentEnd: bool - p.startToken() - p.lexer.documentEndMarker(isDocumentEnd) - if isDocumentEnd: - closeEverything() - p.lexer.bufpos.inc(3) - p.lineEnding() - handleLineEnd(false) - state = fpAfterDocument - else: - raise p.generateError("Unexpected content (expected document end)") - of ' ', '\t', '#': - p.lineEnding() - handleLineEnd(true) - of '\l': p.lexer.bufpos = p.lexer.handleLF(p.lexer.bufpos) - of '\c': p.lexer.bufpos = p.lexer.handleCR(p.lexer.bufpos) - of EndOfFile: - yield endDocEvent() - break - else: - p.startToken() - raise p.generateError("Unexpected content (expected document end)") - of fpAfterDocument: - debug("state: afterDocument") - case p.lexer.buf[p.lexer.bufpos] - of '.': - var isDocumentEnd: bool - p.startToken() - p.lexer.documentEndMarker(isDocumentEnd) - if isDocumentEnd: - p.lexer.bufpos.inc(3) - p.lineEnding() - handleLineEnd(false) - else: - p.initDocValues() - yield startDocEvent() - state = fpBlockObjectStart - of '#': - p.lineEnding() - handleLineEnd(false) - of '\t', ' ': - if not p.consumeLineIfEmpty(p.newlines): - p.indentation = p.lexer.getColNumber(p.lexer.bufpos) - p.initDocValues() - yield startDocEvent() - state = fpBlockObjectStart - of EndOfFile: break - else: - p.initDocValues() - state = fpInitial - of fpFlow: - debug("state: flow") - p.lexer.skipWhitespaceCommentsAndNewlines() - case p.lexer.buf[p.lexer.bufpos] - of '{': - handleFlowItemStart() - yield p.objectStart(yamlStartMap) - flowdepth.inc() - p.lexer.bufpos.inc() - explicitFlowKey = false - of '[': - handleFlowItemStart() - yield p.objectStart(yamlStartSeq) - flowdepth.inc() - p.lexer.bufpos.inc() - of '}': - assert(p.level.kind == fplUnknown) - p.level = p.ancestry.pop() - case p.level.kind - of fplMapValue: - yield emptyScalar(p) - p.level.kind = fplMapKey - of fplMapKey: - if p.tag != yTagQuestionMark or p.anchor != yAnchorNone or - explicitFlowKey: - yield emptyScalar(p) - yield scalarEvent("", yTagQuestionMark, yAnchorNone) - of fplSequence: - p.startToken() - raise p.generateError("Unexpected token (expected ']')") - of fplSinglePairValue: - p.startToken() - raise p.generateError("Unexpected token (expected ']')") - of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail() - p.lexer.bufpos.inc() - leaveFlowLevel() - of ']': - assert(p.level.kind == fplUnknown) - p.level = p.ancestry.pop() - case p.level.kind - of fplSequence: - if p.tag != yTagQuestionMark or p.anchor != yAnchorNone: - yield emptyScalar(p) - of fplSinglePairValue: - yield emptyScalar(p) - p.level = p.ancestry.pop() - yield endMapEvent() - assert(p.level.kind == fplSequence) - of fplMapKey, fplMapValue: - p.startToken() - raise p.generateError("Unexpected token (expected '}')") - of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail() - p.lexer.bufpos.inc() - leaveFlowLevel() - of ',': - assert(p.level.kind == fplUnknown) - p.level = p.ancestry.pop() - case p.level.kind - of fplSequence: yield emptyScalar(p) - of fplMapValue: - yield emptyScalar(p) - p.level.kind = fplMapKey - explicitFlowKey = false - of fplMapKey: - yield emptyScalar(p) - yield scalarEvent("", yTagQuestionMark, yAnchorNone) - explicitFlowKey = false - of fplSinglePairValue: - yield emptyScalar(p) - p.level = p.ancestry.pop() - yield endMapEvent() - assert(p.level.kind == fplSequence) - of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail() - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - p.lexer.bufpos.inc() - of ':': - if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cFlow): - handleFlowItemStart() - handleFlowPlainScalar() - else: - p.level = p.ancestry.pop() - case p.level.kind - of fplSequence: - yield startMapEvent(p.tag, p.anchor) - debug("started single-pair map at " & - (if p.level.indentation == UnknownIndentation: - $p.indentation else: $p.level.indentation)) - p.tag = yTagQuestionMark - p.anchor = yAnchorNone - if p.level.indentation == UnknownIndentation: - p.level.indentation = p.indentation - p.ancestry.add(p.level) - p.level = initLevel(fplSinglePairValue) - yield scalarEvent("") - of fplMapValue, fplSinglePairValue: - p.startToken() - raise p.generateError("Unexpected token (expected ',')") - of fplMapKey: - yield emptyScalar(p) - p.level.kind = fplMapValue - of fplSinglePairKey: - yield emptyScalar(p) - p.level.kind = fplSinglePairValue - of fplUnknown, fplScalar, fplDocument: debugFail() - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - p.lexer.bufpos.inc() - of '\'': - handleFlowItemStart() - p.content.reset() - p.startToken() - p.singleQuotedScalar() - if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark - yield currentScalar(p) - handleObjectEnd(fpFlowAfterObject) - of '"': - handleFlowItemStart() - p.content.reset() - p.startToken() - p.doubleQuotedScalar() - if p.tag == yTagQuestionMark: p.tag = yTagExclamationMark - yield currentScalar(p) - handleObjectEnd(fpFlowAfterObject) - of '!': - handleFlowItemStart() - p.handleTagHandle() - of '&': - handleFlowItemStart() - p.handleAnchor() - of '*': - handleAlias() - state = fpFlowAfterObject - of '?': - if p.lexer.isPlainSafe(p.lexer.bufpos + 1, cFlow): - handleFlowItemStart() - handleFlowPlainScalar() - elif explicitFlowKey: - p.startToken() - raise p.generateError("Duplicate '?' in flow mapping") - elif p.level.kind == fplUnknown: - case p.ancestry[p.ancestry.high].kind - of fplMapKey, fplMapValue, fplDocument: discard - of fplSequence: yield p.objectStart(yamlStartMap, true) - else: - p.startToken() - raise p.generateError("Unexpected token") - explicitFlowKey = true - p.lexer.bufpos.inc() - else: - explicitFlowKey = true - p.lexer.bufpos.inc() - else: - handleFlowItemStart() - handleFlowPlainScalar() - of fpFlowAfterObject: - debug("state: flowAfterObject") - p.lexer.skipWhitespaceCommentsAndNewlines() - case p.lexer.buf[p.lexer.bufpos] - of ']': - case p.level.kind - of fplSequence: discard - of fplMapKey, fplMapValue: - p.startToken() - raise p.generateError("Unexpected token (expected '}')") - of fplSinglePairValue: - p.level = p.ancestry.pop() - assert(p.level.kind == fplSequence) - yield endMapEvent() - of fplScalar, fplUnknown, fplSinglePairKey, fplDocument: debugFail() - p.lexer.bufpos.inc() - leaveFlowLevel() - of '}': - case p.level.kind - of fplMapKey, fplMapValue: discard - of fplSequence, fplSinglePairValue: - p.startToken() - raise p.generateError("Unexpected token (expected ']')") - of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail() - p.lexer.bufpos.inc() - leaveFlowLevel() - of ',': - case p.level.kind - of fplSequence: discard - of fplMapValue: - yield scalarEvent("", yTagQuestionMark, yAnchorNone) - p.level.kind = fplMapKey - explicitFlowKey = false - of fplSinglePairValue: - p.level = p.ancestry.pop() - assert(p.level.kind == fplSequence) - yield endMapEvent() - of fplMapKey: explicitFlowKey = false - of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail() - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - state = fpFlow - p.lexer.bufpos.inc() - of ':': - case p.level.kind - of fplSequence, fplMapKey: - p.startToken() - raise p.generateError("Unexpected token (expected ',')") - of fplMapValue, fplSinglePairValue: discard - of fplUnknown, fplScalar, fplSinglePairKey, fplDocument: debugFail() - p.ancestry.add(p.level) - p.level = initLevel(fplUnknown) - state = fpFlow - p.lexer.bufpos.inc() - of '#': - p.lineEnding() - handleLineEnd(true) - of EndOfFile: - p.startToken() - raise p.generateError("Unclosed flow content") - else: - p.startToken() - raise p.generateError("Unexpected content (expected flow indicator)") - try: result = initYamlStream(backend) - except Exception: debugFail() # compiler error \ No newline at end of file diff --git a/private/internal.nim b/private/internal.nim new file mode 100644 index 0000000..af74036 --- /dev/null +++ b/private/internal.nim @@ -0,0 +1,29 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +template internalError*(s: string) = + when not defined(release): + let ii = instantiationInfo() + echo "[NimYAML] Error in file ", ii.filename, " at line ", ii.line, ":" + echo s + when not defined(JS): + echo "[NimYAML] Stacktrace:" + try: writeStackTrace() + except: discard + echo "[NimYAML] Please report this bug." + quit 1 +template yAssert*(e: typed) = + when not defined(release): + if not e: + let ii = instantiationInfo() + echo "[NimYAML] Error in file ", ii.filename, " at line ", ii.line, ":" + echo "assertion failed!" + when not defined(JS): + echo "[NimYAML] Stacktrace:" + try: writeStackTrace() + except: discard + echo "[NimYAML] Please report this bug." + quit 1 \ No newline at end of file diff --git a/private/lex.nim b/private/lex.nim new file mode 100644 index 0000000..048e7e1 --- /dev/null +++ b/private/lex.nim @@ -0,0 +1,1189 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2015 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +import lexbase, streams, strutils, unicode +when defined(yamlDebug): + import terminal + export terminal + +type + StringSource* = object + src: string + pos: int + line, lineStart: int + + SourceProvider* = concept c + advance(c) is char + lexCR(c) + lexLF(c) + + YamlLexerObj* = object + cur*: LexerToken + curStartPos*: tuple[line, column: int] + # ltScalarPart, ltQuotedScalar, ltYamlVersion, ltTagShorthand, ltTagUri, + # ltLiteralTag, ltTagHandle, ltAnchor, ltAlias + buf*: string not nil + # ltIndentation + indentation*: int + # ltTagHandle + shorthandEnd*: int + + # may be modified from outside; will be consumed at plain scalar starts + newlines*: int + + # internals + source: pointer + inFlow: bool + literalEndIndent: int + nextState, lineStartState, inlineState, insideLineImpl, insideDocImpl, + insideFlowImpl, outsideDocImpl: LexerState + blockScalarIndent: int + folded: bool + chomp: ChompType + c: char + tokenLineGetter: proc(lex: YamlLexer, pos: tuple[line, column: int], + marker: bool): string {.raises: [].} + searchColonImpl: proc(lex: YamlLexer): bool + + YamlLexer* = ref YamlLexerObj + + YamlLexerError* = object of Exception + line*, column*: int + lineContent*: string + + LexerState = proc(lex: YamlLexer): bool {.raises: YamlLexerError, locks: 0, + gcSafe.} + + LexerToken* = enum + ltYamlDirective, ltYamlVersion, ltTagDirective, ltTagShorthand, + ltTagUri, ltUnknownDirective, ltUnknownDirectiveParams, ltEmptyLine, + ltDirectivesEnd, ltDocumentEnd, ltStreamEnd, ltIndentation, ltQuotedScalar, + ltScalarPart, ltBlockScalarHeader, ltBlockScalar, ltSeqItemInd, ltMapKeyInd, + ltMapValInd, ltBraceOpen, ltBraceClose, ltBracketOpen, ltBracketClose, + ltComma, ltLiteralTag, ltTagHandle, ltAnchor, ltAlias + + ChompType* = enum + ctKeep, ctClip, ctStrip + +# consts + +const + space = {' ', '\t'} + lineEnd = {'\l', '\c', EndOfFile} + spaceOrLineEnd = {' ', '\t', '\l', '\c', EndOfFile} + digits = {'0'..'9'} + flowIndicators = {'[', ']', '{', '}', ','} + uriChars = {'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', + '@', '&', '-', '=', '+', '$', '_', '.', '~', '*', '\'', '(', ')'} + + UTF8NextLine = toUTF8(0x85.Rune) + UTF8NonBreakingSpace = toUTF8(0xA0.Rune) + UTF8LineSeparator = toUTF8(0x2028.Rune) + UTF8ParagraphSeparator = toUTF8(0x2029.Rune) + + UnknownIndentation* = int.low + +# lexer backend implementations + +template blSource(lex: YamlLexer): var BaseLexer = + (cast[ptr BaseLexer](lex.source))[] +template sSource(lex: YamlLexer): var StringSource = + (cast[ptr StringSource](lex.source))[] + +proc advance(lex: YamlLexer, t: typedesc[BaseLexer], step: int = 1) {.inline.} = + lex.blSource.bufpos.inc(step) + lex.c = lex.blSource.buf[lex.blSource.bufpos] + +proc advance(lex: YamlLexer, t: typedesc[StringSource], step: int = 1) + {.inline.} = + lex.sSource.pos.inc(step) + if lex.sSource.pos >= lex.sSource.src.len: lex.c = EndOfFile + else: lex.c = lex.sSource.src[lex.sSource.pos] + +template lexCR(lex: YamlLexer, t: typedesc[BaseLexer]) = + try: lex.blSource.bufpos = lex.blSource.handleCR(lex.blSource.bufpos) + except: + var e = generateError[T](lex, "Encountered stream error: " & + getCurrentExceptionMsg()) + e.parent = getCurrentException() + raise e + lex.c = lex.blSource.buf[lex.blSource.bufpos] + +template lexCR(lex: YamlLexer, t: typedesc[StringSource]) = + lex.sSource.pos.inc() + if lex.sSource.src[lex.sSource.pos] == '\l': lex.sSource.pos.inc() + lex.sSource.lineStart = lex.sSource.pos + lex.sSource.line.inc() + lex.c = lex.sSource.src[lex.sSource.pos] + +template lexLF(lex: YamlLexer, t: typedesc[BaseLexer]) = + try: lex.blSource.bufpos = lex.blSource.handleLF(lex.blSource.bufpos) + except: + var e = generateError[T](lex, "Encountered stream error: " & + getCurrentExceptionMsg()) + e.parent = getCurrentException() + raise e + lex.c = lex.blSource.buf[lex.blSource.bufpos] + +template lexLF(lex: YamlLexer, t: typedesc[StringSource]) = + lex.sSource.pos.inc() + lex.sSource.lineStart = lex.sSource.pos + lex.sSource.line.inc() + lex.c = lex.sSource.src[lex.sSource.pos] + +template lineNumber(lex: YamlLexer, t: typedesc[BaseLexer]): int = + lex.blSource.lineNumber + +template lineNumber(lex: YamlLexer, t: typedesc[StringSource]): int = + lex.sSource.line + +template columnNumber(lex: YamlLexer, t: typedesc[BaseLexer]): int = + lex.blSource.getColNumber(lex.blSource.bufpos) + 1 + +template columnNumber(lex: YamlLexer, t: typedesc[StringSource]): int = + lex.sSource.pos - lex.sSource.lineStart + 1 + +template currentLine(lex: YamlLexer, t: typedesc[BaseLexer]): string = + lex.blSource.getCurrentLine(true) + +template currentLine(lex: YamlLexer, t: typedesc[StringSource]): string = + var result = "" + var i = lex.sSource.lineStart + while lex.sSource.src[i] notin lineEnd: + result.add(lex.sSource.src[i]) + inc(i) + result.add("\n" & spaces(lex.columnNumber(t) - 1) & "^\n") + result + +proc nextIsPlainSafe(lex: YamlLexer, t: typedesc[BaseLexer], inFlow: bool): + bool {.inline.} = + case lex.blSource.buf[lex.blSource.bufpos + 1] + of spaceOrLineEnd: result = false + of flowIndicators: result = not inFlow + else: result = true + +proc nextIsPlainSafe(lex: YamlLexer, t: typedesc[StringSource], + inFlow: bool): bool {.inline.} = + case lex.sSource.src[lex.sSource.pos + 1] + of spaceOrLineEnd: result = false + of flowIndicators: result = not inFlow + else: result = true + +proc getPos(lex: YamlLexer, t: typedesc[BaseLexer]): int = lex.blSource.bufpos +proc getPos(lex: YamlLexer, t: typedesc[StringSource]): int = lex.sSource.pos + +proc at(lex: YamlLexer, t: typedesc[BaseLexer], pos: int): char {.inline.} = + lex.blSource.buf[pos] + +proc at(lex: YamlLexer, t: typedesc[StringSource], pos: int): char {.inline.} = + lex.sSource.src[pos] + +proc mark(lex: YamlLexer, t: typedesc[BaseLexer]): int = lex.blSource.bufpos +proc mark(lex: YamlLexer, t: typedesc[StringSource]): int = lex.sSource.pos + +proc afterMark(lex: YamlLexer, t: typedesc[BaseLexer], m: int): int {.inline.} = + lex.blSource.bufpos - m + +proc afterMark(lex: YamlLexer, t: typedesc[StringSource], m: int): + int {.inline.} = + lex.sSource.pos - m + +proc lineWithMarker(lex: YamlLexer, pos: tuple[line, column: int], + t: typedesc[BaseLexer], marker: bool): string = + if pos.line == lex.blSource.lineNumber: + result = lex.blSource.getCurrentLine(false) + if marker: result.add(spaces(pos.column - 1) & "^\n") + else: result = "" + +proc lineWithMarker(lex: YamlLexer, pos: tuple[line, column: int], + t: typedesc[StringSource], marker: bool): string = + var + lineStartIndex = lex.sSource.pos + lineEndIndex: int + curLine = lex.sSource.line + if pos.line == curLine: + lineEndIndex = lex.sSource.pos + while lex.sSource.src[lineEndIndex] notin lineEnd: inc(lineEndIndex) + while true: + while lineStartIndex >= 0 and lex.sSource.src[lineStartIndex] notin lineEnd: + dec(lineStartIndex) + if curLine == pos.line: + inc(lineStartIndex) + break + let wasLF = lex.sSource.src[lineStartIndex] == '\l' + lineEndIndex = lineStartIndex + dec(lineStartIndex) + if lex.sSource.src[lineStartIndex] == '\c' and wasLF: + dec(lineStartIndex) + dec(lineEndIndex) + dec(curLine) + result = lex.sSource.src.substr(lineStartIndex, lineEndIndex - 1) & "\n" + if marker: result.add(spaces(pos.column - 1) & "^\n") + +# lexer states + +{.push raises: YamlLexerError, gcSafe, locks: 0.} +proc outsideDoc[T](lex: YamlLexer): bool +proc yamlVersion[T](lex: YamlLexer): bool +proc tagShorthand[T](lex: YamlLexer): bool +proc tagUri[T](lex: YamlLexer): bool +proc unknownDirParams[T](lex: YamlLexer): bool +proc expectLineEnd[T](lex: YamlLexer): bool +proc possibleDirectivesEnd[T](lex: YamlLexer): bool +proc possibleDocumentEnd[T](lex: YamlLexer): bool +proc afterSeqInd[T](lex: YamlLexer): bool +proc insideDoc[T](lex: YamlLexer): bool +proc insideFlow[T](lex: YamlLexer): bool +proc insideLine[T](lex: YamlLexer): bool +proc plainScalarPart[T](lex: YamlLexer): bool +proc blockScalarHeader[T](lex: YamlLexer): bool +proc blockScalar[T](lex: YamlLexer): bool +proc indentationAfterBlockScalar[T](lex: YamlLexer): bool +proc dirEndAfterBlockScalar[T](lex: YamlLexer): bool +proc docEndAfterBlockScalar[T](lex: YamlLexer): bool +proc tagHandle[T](lex: YamlLexer): bool +proc anchor[T](lex: YamlLexer): bool +proc alias[T](lex: YamlLexer): bool +proc streamEnd[T](lex: YamlLexer): bool +{.pop.} + +# implementation + +template debug(message: string) {.dirty.} = + when defined(yamlDebug): + try: styledWriteLine(stdout, fgBlue, message) + except IOError: discard + +proc generateError[T](lex: YamlLexer, message: string): + ref YamlLexerError {.raises: [].} = + result = newException(YamlLexerError, message) + result.line = lex.lineNumber(T) + result.column = lex.columnNumber(T) + result.lineContent = lex.currentLine(T) + +proc startToken[T](lex: YamlLexer) {.inline.} = + lex.curStartPos = (lex.lineNumber(T), lex.columnNumber(T)) + +proc directiveName[T](lex: YamlLexer) = + while lex.c notin spaceOrLineEnd: + lex.buf.add(lex.c) + lex.advance(T) + +proc consumeNewlines(lex: YamlLexer) {.inline, raises: [].} = + case lex.newlines + of 0: return + of 1: lex.buf.add(' ') + else: lex.buf.add(repeat('\l', lex.newlines - 1)) + lex.newlines = 0 + +proc yamlVersion[T](lex: YamlLexer): bool = + debug("lex: yamlVersion") + while lex.c in space: lex.advance(T) + if lex.c notin digits: + raise generateError[T](lex, "Invalid YAML version number") + startToken[T](lex) + lex.buf.add(lex.c) + lex.advance(T) + while lex.c in digits: + lex.buf.add(lex.c) + lex.advance(T) + if lex.c != '.': raise generateError[T](lex, "Invalid YAML version number") + lex.buf.add('.') + lex.advance(T) + if lex.c notin digits: + raise generateError[T](lex, "Invalid YAML version number") + lex.buf.add(lex.c) + lex.advance(T) + while lex.c in digits: + lex.buf.add(lex.c) + lex.advance(T) + if lex.c notin spaceOrLineEnd: + raise generateError[T](lex, "Invalid YAML version number") + lex.cur = ltYamlVersion + result = true + lex.nextState = expectLineEnd[T] + +proc tagShorthand[T](lex: YamlLexer): bool = + debug("lex: tagShorthand") + while lex.c in space: lex.advance(T) + if lex.c != '!': + raise generateError[T](lex, "Tag shorthand must start with a '!'") + startToken[T](lex) + lex.buf.add(lex.c) + lex.advance(T) + + if lex.c in spaceOrLineEnd: discard + else: + while lex.c != '!': + case lex.c + of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '-': + lex.buf.add(lex.c) + lex.advance(T) + else: raise generateError[T](lex, "Illegal character in tag shorthand") + lex.buf.add(lex.c) + lex.advance(T) + if lex.c notin spaceOrLineEnd: + raise generateError[T](lex, "Missing space after tag shorthand") + lex.cur = ltTagShorthand + result = true + lex.nextState = tagUri[T] + +proc tagUri[T](lex: YamlLexer): bool = + debug("lex: tagUri") + while lex.c in space: lex.advance(T) + startToken[T](lex) + if lex.c == '!': + lex.buf.add(lex.c) + lex.advance(T) + while true: + case lex.c + of spaceOrLineEnd: break + of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', '@', '&', + '-', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')': + lex.buf.add(lex.c) + lex.advance(T) + else: raise generateError[T](lex, "Invalid character in tag uri: " & + escape("" & lex.c)) + lex.cur = ltTagUri + result = true + lex.nextState = expectLineEnd[T] + +proc unknownDirParams[T](lex: YamlLexer): bool = + debug("lex: unknownDirParams") + while lex.c in space: lex.advance(T) + startToken[T](lex) + while lex.c notin lineEnd + {'#'}: + lex.buf.add(lex.c) + lex.advance(T) + lex.cur = ltUnknownDirectiveParams + result = true + lex.nextState = expectLineEnd[T] + +proc expectLineEnd[T](lex: YamlLexer): bool = + debug("lex: expectLineEnd") + result = false + while lex.c in space: lex.advance(T) + while true: + case lex.c + of '#': + lex.advance(T) + while lex.c notin lineEnd: lex.advance(T) + of EndOfFile: + lex.nextState = streamEnd[T] + break + of '\l': + lex.lexLF(T) + lex.nextState = lex.lineStartState + break + of '\c': + lex.lexCR(T) + lex.nextState = lex.lineStartState + break + else: + raise generateError[T](lex, + "Unexpected character (expected line end): " & escape("" & lex.c)) + +proc possibleDirectivesEnd[T](lex: YamlLexer): bool = + debug("lex: possibleDirectivesEnd") + lex.indentation = 0 + lex.lineStartState = lex.insideDocImpl # could be insideDoc[T] + lex.advance(T) + if lex.c == '-': + lex.advance(T) + if lex.c == '-': + lex.advance(T) + if lex.c in spaceOrLineEnd: + lex.cur = ltDirectivesEnd + while lex.c in space: lex.advance(T) + lex.nextState = lex.insideLineImpl + return true + lex.consumeNewlines() + lex.buf.add('-') + else: lex.consumeNewlines() + lex.buf.add('-') + elif lex.c in spaceOrLineEnd: + lex.cur = ltIndentation + lex.nextState = afterSeqInd[T] + return true + else: lex.consumeNewlines() + lex.buf.add('-') + lex.cur = ltIndentation + lex.nextState = plainScalarPart[T] + result = true + +proc afterSeqInd[T](lex: YamlLexer): bool = + result = true + lex.cur = ltSeqItemInd + if lex.c notin lineEnd: + lex.advance(T) + while lex.c in space: lex.advance(T) + lex.nextState = lex.insideLineImpl + +proc possibleDocumentEnd[T](lex: YamlLexer): bool = + debug("lex: possibleDocumentEnd") + lex.advance(T) + if lex.c == '.': + lex.advance(T) + if lex.c == '.': + lex.advance(T) + if lex.c in spaceOrLineEnd: + lex.cur = ltDocumentEnd + lex.nextState = expectLineEnd[T] + lex.lineStartState = lex.outsideDocImpl + return true + lex.consumeNewlines() + lex.buf.add('.') + else: lex.consumeNewlines() + lex.buf.add('.') + else: lex.consumeNewlines() + lex.buf.add('.') + lex.nextState = plainScalarPart[T] + result = false + +proc outsideDoc[T](lex: YamlLexer): bool = + debug("lex: outsideDoc") + startToken[T](lex) + case lex.c + of '%': + lex.advance(T) + directiveName[T](lex) + case lex.buf + of "YAML": + lex.cur = ltYamlDirective + lex.buf.setLen(0) + lex.nextState = yamlVersion[T] + of "TAG": + lex.buf.setLen(0) + lex.cur = ltTagDirective + lex.nextState = tagShorthand[T] + else: + lex.cur = ltUnknownDirective + lex.nextState = unknownDirParams[T] + return true + of '-': + lex.nextState = possibleDirectivesEnd[T] + return false + of '.': + lex.indentation = 0 + if possibleDocumentEnd[T](lex): return true + of spaceOrLineEnd + {'#'}: + lex.indentation = 0 + while lex.c == ' ': + lex.indentation.inc() + lex.advance(T) + if lex.c in spaceOrLineEnd + {'#'}: + lex.nextState = expectLineEnd[T] + return false + lex.nextState = insideLine[T] + else: + lex.indentation = 0 + lex.nextState = insideLine[T] + lex.lineStartState = insideDoc[T] + lex.cur = ltIndentation + result = true + +proc insideDoc[T](lex: YamlLexer): bool = + debug("lex: insideDoc") + startToken[T](lex) + lex.indentation = 0 + case lex.c + of '-': + lex.nextState = possibleDirectivesEnd[T] + return false + of '.': lex.nextState = possibleDocumentEnd[T] + of spaceOrLineEnd: + while lex.c == ' ': + lex.indentation.inc() + lex.advance(T) + while lex.c in space: lex.advance(T) + case lex.c + of lineEnd: + lex.cur = ltEmptyLine + lex.nextState = expectLineEnd[T] + return true + else: + lex.nextState = lex.inlineState + else: lex.nextState = lex.inlineState + lex.cur = ltIndentation + result = true + +proc insideFlow[T](lex: YamlLexer): bool = + debug("lex: insideFlow") + startToken[T](lex) + while lex.c in space: lex.advance(T) + if lex.c in lineEnd + {'#'}: + lex.cur = ltEmptyLine + lex.nextState = expectLineEnd[T] + return true + lex.nextState = insideLine[T] + result = false + +proc possibleIndicatorChar[T](lex: YamlLexer, indicator: LexerToken, + jsonContext: bool = false): bool = + startToken[T](lex) + if not(jsonContext) and lex.nextIsPlainSafe(T, lex.inFlow): + lex.consumeNewlines() + lex.nextState = plainScalarPart[T] + result = false + else: + lex.cur = indicator + result = true + lex.advance(T) + while lex.c in space: lex.advance(T) + if lex.c in lineEnd: + lex.nextState = expectLineEnd[T] + +proc flowIndicator[T](lex: YamlLexer, indicator: LexerToken): bool {.inline.} = + startToken[T](lex) + lex.cur = indicator + lex.advance(T) + while lex.c in space: lex.advance(T) + if lex.c in lineEnd + {'#'}: + lex.nextState = expectLineEnd[T] + result = true + +proc addMultiple(s: var string, c: char, num: int) {.raises: [], inline.} = + for i in 1..num: s.add(c) + +proc processQuotedWhitespace[T](lex: YamlLexer, newlines: var int) = + block outer: + let beforeSpace = lex.buf.len + while true: + case lex.c + of ' ', '\t': lex.buf.add(lex.c) + of '\l': + lex.lexLF(T) + break + of '\c': + lex.lexCR(T) + break + else: break outer + lex.advance(T) + lex.buf.setLen(beforeSpace) + while true: + case lex.c + of ' ', '\t': discard + of '\l': + lex.lexLF(T) + newlines.inc() + continue + of '\c': + lex.lexCR(T) + newlines.inc() + continue + else: + if newlines == 0: discard + elif newlines == 1: lex.buf.add(' ') + else: lex.buf.addMultiple('\l', newlines - 1) + break + lex.advance(T) + +proc singleQuotedScalar[T](lex: YamlLexer) = + debug("lex: singleQuotedScalar") + startToken[T](lex) + lex.advance(T) + while true: + case lex.c + of '\'': + lex.advance(T) + if lex.c == '\'': lex.buf.add('\'') + else: break + of EndOfFile: raise generateError[T](lex, "Unfinished single quoted string") + of '\l', '\c', '\t', ' ': + var newlines = 1 + processQuotedWhitespace[T](lex, newlines) + continue + else: lex.buf.add(lex.c) + lex.advance(T) + while lex.c in space: lex.advance(T) + if lex.c in lineEnd + {'#'}: + lex.nextState = expectLineEnd[T] + +proc unicodeSequence[T](lex: YamlLexer, length: int) = + debug("lex: unicodeSequence") + var unicodeChar = 0.int + for i in countup(0, length - 1): + lex.advance(T) + let digitPosition = length - i - 1 + case lex.c + of EndOFFile, '\l', '\c': + raise generateError[T](lex, "Unfinished unicode escape sequence") + of '0' .. '9': + unicodeChar = unicodechar or (int(lex.c) - 0x30) shl (digitPosition * 4) + of 'A' .. 'F': + unicodeChar = unicodechar or (int(lex.c) - 0x37) shl (digitPosition * 4) + of 'a' .. 'f': + unicodeChar = unicodechar or (int(lex.c) - 0x57) shl (digitPosition * 4) + else: + raise generateError[T](lex, + "Invalid character in unicode escape sequence: " & + escape("" & lex.c)) + lex.buf.add(toUTF8(Rune(unicodeChar))) + +proc doubleQuotedScalar[T](lex: YamlLexer) = + debug("lex: doubleQuotedScalar") + startToken[T](lex) + lex.advance(T) + while true: + case lex.c + of EndOfFile: + raise generateError[T](lex, "Unfinished double quoted string") + of '\\': + lex.advance(T) + case lex.c + of EndOfFile: + raise generateError[T](lex, "Unfinished escape sequence") + of '0': lex.buf.add('\0') + of 'a': lex.buf.add('\x07') + of 'b': lex.buf.add('\x08') + of '\t', 't': lex.buf.add('\t') + of 'n': lex.buf.add('\l') + of 'v': lex.buf.add('\v') + of 'f': lex.buf.add('\f') + of 'r': lex.buf.add('\c') + of 'e': lex.buf.add('\e') + of ' ': lex.buf.add(' ') + of '"': lex.buf.add('"') + of '/': lex.buf.add('/') + of '\\': lex.buf.add('\\') + of 'N': lex.buf.add(UTF8NextLine) + of '_': lex.buf.add(UTF8NonBreakingSpace) + of 'L': lex.buf.add(UTF8LineSeparator) + of 'P': lex.buf.add(UTF8ParagraphSeparator) + of 'x': unicodeSequence[T](lex, 2) + of 'u': unicodeSequence[T](lex, 4) + of 'U': unicodeSequence[T](lex, 8) + of '\l', '\c': + var newlines = 0 + processQuotedWhitespace[T](lex, newlines) + continue + else: raise generateError[T](lex, "Illegal character in escape sequence") + of '"': + lex.advance(T) + break + of '\l', '\c', '\t', ' ': + var newlines = 1 + processQuotedWhitespace[T](lex, newlines) + continue + else: lex.buf.add(lex.c) + lex.advance(T) + while lex.c in space: lex.advance(T) + if lex.c in lineEnd + {'#'}: + lex.nextState = expectLineEnd[T] + +proc insideLine[T](lex: YamlLexer): bool = + debug("lex: insideLine") + case lex.c + of ':': + result = possibleIndicatorChar[T](lex, ltMapValInd, + lex.inFlow and + lex.cur in [ltBraceClose, ltBracketClose, ltQuotedScalar]) + of '?': result = possibleIndicatorChar[T](lex, ltMapKeyInd) + of '-': result = possibleIndicatorChar[T](lex, ltSeqItemInd) + of lineEnd + {'#'}: + result = false + lex.nextState = expectLineEnd[T] + of '\"': + doubleQuotedScalar[T](lex) + lex.cur = ltQuotedScalar + result = true + of '\'': + singleQuotedScalar[T](lex) + lex.cur = ltQuotedScalar + result = true + of '>', '|': + startToken[T](lex) + lex.consumeNewlines() + if lex.inFlow: lex.nextState = plainScalarPart[T] + else: lex.nextState = blockScalarHeader[T] + result = false + of '{': result = flowIndicator[T](lex, ltBraceOpen) + of '}': result = flowIndicator[T](lex, ltBraceClose) + of '[': result = flowIndicator[T](lex, ltBracketOpen) + of ']': result = flowIndicator[T](lex, ltBracketClose) + of ',': result = flowIndicator[T](lex, ltComma) + of '!': + lex.nextState = tagHandle[T] + result = false + of '&': + lex.nextState = anchor[T] + result = false + of '*': + lex.nextState = alias[T] + result = false + of '@', '`': + raise generateError[T](lex, + "Reserved characters cannot start a plain scalar") + else: + startToken[T](lex) + lex.consumeNewlines() + lex.nextState = plainScalarPart[T] + result = false + +proc plainScalarPart[T](lex: YamlLexer): bool = + debug("lex: plainScalarPart") + block outer: + while true: + lex.buf.add(lex.c) + lex.advance(T) + case lex.c + of space: + let lenBeforeSpace = lex.buf.len() + while true: + lex.buf.add(lex.c) + lex.advance(T) + case lex.c + of lineEnd + {'#'}: + lex.buf.setLen(lenBeforeSpace) + lex.nextState = expectLineEnd[T] + break outer + of ':': + if lex.nextIsPlainSafe(T, lex.inFlow): break + else: + lex.buf.setLen(lenBeforeSpace) + lex.nextState = lex.insideLineImpl # could be insideLine[T] + break outer + of flowIndicators: + if lex.inFlow: + lex.buf.setLen(lenBeforeSpace) + lex.nextState = lex.insideLineImpl # could be insideLine[T] + break outer + else: + lex.buf.add(lex.c) + lex.advance(T) + break + of space: discard + else: break + of lineEnd: + lex.nextState = expectLineEnd[T] + break + of flowIndicators: + if lex.inFlow: + lex.nextState = lex.insideLineImpl # could be insideLine[T] + break + of ':': + if not lex.nextIsPlainSafe(T, lex.inFlow): + lex.nextState = lex.insideLineImpl # could be insideLine[T] + break outer + else: discard + lex.cur = ltScalarPart + result = true + +proc blockScalarHeader[T](lex: YamlLexer): bool = + debug("lex: blockScalarHeader") + lex.chomp = ctClip + lex.blockScalarIndent = UnknownIndentation + lex.folded = lex.c == '>' + startToken[T](lex) + while true: + lex.advance(T) + case lex.c + of '+': + if lex.chomp != ctClip: + raise generateError[T](lex, "Only one chomping indicator is allowed") + lex.chomp = ctKeep + of '-': + if lex.chomp != ctClip: + raise generateError[T](lex, "Only one chomping indicator is allowed") + lex.chomp = ctStrip + of '1'..'9': + if lex.blockScalarIndent != UnknownIndentation: + raise generateError[T](lex, "Only one indentation indicator is allowed") + lex.blockScalarIndent = ord(lex.c) - ord('\x30') + of spaceOrLineEnd: break + else: + raise generateError[T](lex, + "Illegal character in block scalar header: '" & escape("" & lex.c) & + '\'') + lex.nextState = expectLineEnd[T] + lex.lineStartState = blockScalar[T] + lex.cur = ltBlockScalarHeader + result = true + +proc blockScalarAfterLineStart[T](lex: YamlLexer, + recentWasMoreIndented: var bool): bool = + if lex.indentation < lex.blockScalarIndent: + lex.nextState = indentationAfterBlockScalar[T] + return false + + if lex.folded and not recentWasMoreIndented: lex.consumeNewlines() + else: + recentWasMoreIndented = false + lex.buf.add(repeat('\l', lex.newlines)) + lex.newlines = 0 + result = true + +proc blockScalarLineStart[T](lex: YamlLexer, recentWasMoreIndented: var bool): + bool = + while true: + case lex.c + of '-': + if lex.indentation < lex.blockScalarIndent: + lex.nextState = indentationAfterBlockScalar[T] + return false + discard possibleDirectivesEnd[T](lex) + case lex.cur + of ltDirectivesEnd: + lex.nextState = dirEndAfterBlockScalar[T] + return false + of ltIndentation: + if lex.nextState == afterSeqInd[T]: + lex.consumeNewlines() + lex.buf.add("- ") + else: discard + break + of '.': + if lex.indentation < lex.blockScalarIndent: + lex.nextState = indentationAfterBlockScalar[T] + return false + if possibleDocumentEnd[T](lex): + lex.nextState = docEndAfterBlockScalar[T] + return false + break + of spaceOrLineEnd: + while lex.c == ' ' and lex.indentation < lex.blockScalarIndent: + lex.indentation.inc() + lex.advance(T) + case lex.c + of '\l': + lex.newlines.inc() + lex.lexLF(T) + lex.indentation = 0 + of '\c': + lex.newlines.inc() + lex.lexCR(T) + lex.indentation = 0 + of EndOfFile: + lex.nextState = streamEnd[T] + return false + of ' ', '\t': + recentWasMoreIndented = true + lex.buf.add(repeat('\l', lex.newlines)) + lex.newlines = 0 + return true + else: break + else: break + result = blockScalarAfterLineStart[T](lex, recentWasMoreIndented) + +proc blockScalar[T](lex: YamlLexer): bool = + debug("lex: blockScalar") + block outer: + var recentWasMoreIndented = true + if lex.blockScalarIndent == UnknownIndentation: + while true: + lex.blockScalarIndent = 0 + while lex.c == ' ': + lex.blockScalarIndent.inc() + lex.advance(T) + case lex.c + of '\l': + lex.lexLF(T) + lex.newlines.inc() + of '\c': + lex.lexCR(T) + lex.newlines.inc() + of EndOfFile: + lex.nextState = streamEnd[T] + break outer + else: + if lex.blockScalarIndent <= lex.indentation: + lex.indentation = lex.blockScalarIndent + lex.nextState = indentationAfterBlockScalar[T] + break outer + lex.indentation = lex.blockScalarIndent + break + else: + lex.blockScalarIndent += lex.indentation + if lex.c notin {'.', '-'} or lex.indentation == 0: + if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer + else: + if not blockScalarAfterLineStart[T](lex, recentWasMoreIndented): + break outer + while true: + while lex.c notin lineEnd: + lex.buf.add(lex.c) + lex.advance(T) + if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer + + debug("lex: leaving block scalar at indentation " & $lex.indentation) + case lex.chomp + of ctStrip: discard + of ctClip: + if lex.buf.len > 0: lex.buf.add('\l') + of ctKeep: lex.buf.add(repeat('\l', lex.newlines)) + lex.newlines = 0 + lex.lineStartState = insideDoc[T] + lex.cur = ltBlockScalar + result = true + +proc indentationAfterBlockScalar[T](lex: YamlLexer): bool = + if lex.indentation == 0: + lex.nextState = lex.insideDocImpl + elif lex.c == '#': + lex.nextState = expectLineEnd[T] + result = false + else: + lex.cur = ltIndentation + result = true + lex.nextState = lex.insideLineImpl + +proc dirEndAfterBlockScalar[T](lex: YamlLexer): bool = + lex.cur = ltDirectivesEnd + while lex.c in space: lex.advance(T) + lex.nextState = lex.insideLineImpl + result = true + +proc docEndAfterBlockScalar[T](lex: YamlLexer): bool = + lex.cur = ltDocumentEnd + lex.nextState = expectLineEnd[T] + lex.lineStartState = lex.outsideDocImpl + result = true + +proc byteSequence[T](lex: YamlLexer) = + debug("lex: byteSequence") + var charCode = 0.int8 + for i in 0 .. 1: + lex.advance(T) + let digitPosition = int8(1 - i) + case lex.c + of EndOfFile, '\l', 'r': + raise generateError[T](lex, "Unfinished octet escape sequence") + of '0' .. '9': + charCode = charCode or (int8(lex.c) - 0x30.int8) shl (digitPosition * 4) + of 'A' .. 'F': + charCode = charCode or (int8(lex.c) - 0x37.int8) shl (digitPosition * 4) + of 'a' .. 'f': + charCode = charCode or (int8(lex.c) - 0x57.int8) shl (digitPosition * 4) + else: + raise generateError[T](lex, "Invalid character in octet escape sequence") + lex.buf.add(char(charCode)) + +proc tagHandle[T](lex: YamlLexer): bool = + debug("lex: tagHandle") + startToken[T](lex) + lex.advance(T) + if lex.c == '<': + lex.advance(T) + if lex.c == '!': + lex.buf.add('!') + lex.advance(T) + while true: + case lex.c + of spaceOrLineEnd: raise generateError[T](lex, "Unclosed verbatim tag") + of '%': byteSequence[T](lex) + of uriChars + {','}: lex.buf.add(lex.c) + of '>': break + else: raise generateError[T](lex, "Illegal character in verbatim tag") + lex.advance(T) + lex.advance(T) + lex.cur = ltLiteralTag + else: + lex.shorthandEnd = 0 + let m = lex.mark(T) + lex.buf.add('!') + while true: + case lex.c + of spaceOrLineEnd: break + of '!': + if lex.shorthandEnd != 0: + raise generateError[T](lex, "Illegal character in tag suffix") + lex.shorthandEnd = lex.afterMark(T, m) + 1 + lex.buf.add('!') + of ',': + if lex.shorthandEnd > 0: break # ',' after shorthand is flow indicator + lex.buf.add(',') + of '%': + if lex.shorthandEnd == 0: + raise generateError[T](lex, "Illegal character in tag handle") + byteSequence[T](lex) + of uriChars: lex.buf.add(lex.c) + else: raise generateError[T](lex, "Illegal character in tag handle") + lex.advance(T) + lex.cur = ltTagHandle + while lex.c in space: lex.advance(T) + if lex.c in lineEnd: lex.nextState = expectLineEnd[T] + else: lex.nextState = lex.insideLineImpl # could be insideLine[T] + result = true + +proc anchorName[T](lex: YamlLexer) = + debug("lex: anchorName") + startToken[T](lex) + while true: + lex.advance(T) + case lex.c + of spaceOrLineEnd, '[', ']', '{', '}', ',': break + else: lex.buf.add(lex.c) + while lex.c in space: lex.advance(T) + if lex.c in lineEnd: lex.nextState = expectLineEnd[T] + else: lex.nextState = lex.insideLineImpl # could be insideLine[T] + +proc anchor[T](lex: YamlLexer): bool = + debug("lex: anchor") + anchorName[T](lex) + lex.cur = ltAnchor + result = true + +proc alias[T](lex: YamlLexer): bool = + debug("lex: alias") + anchorName[T](lex) + lex.cur = ltAlias + result = true + +proc streamEnd[T](lex: YamlLexer): bool = + debug("lex: streamEnd") + startToken[T](lex) + lex.cur = ltStreamEnd + result = true + +proc tokenLine[T](lex: YamlLexer, pos: tuple[line, column: int], marker: bool): + string = + result = lex.lineWithMarker(pos, T, marker) + +proc searchColon[T](lex: YamlLexer): bool = + var flowDepth = if lex.cur in [ltBraceOpen, ltBracketOpen]: 1 else: 0 + let start = lex.getPos(T) + var + peek = start + recentAllowsAdjacent = lex.cur == ltQuotedScalar + result = false + + proc skipPlainScalarContent(lex: YamlLexer) {.inline.} = + while true: + inc(peek) + case lex.at(T, peek) + of ']', '}', ',': + if flowDepth > 0 or lex.inFlow: break + of '#': + if lex.at(T, peek - 1) in space: break + of ':': + if lex.at(T, peek + 1) in spaceOrLineEnd: break + of lineEnd: break + else: discard + + while peek < start + 1024: + case lex.at(T, peek) + of ':': + if flowDepth == 0: + if recentAllowsAdjacent or lex.at(T, peek + 1) in spaceOrLineEnd: + result = true + break + lex.skipPlainScalarContent() + continue + of '{', '[': inc(flowDepth) + of '}', ']': + dec(flowDepth) + if flowDepth < 0: + if lex.inFlow: break + else: + flowDepth = 0 + lex.skipPlainScalarContent() + continue + recentAllowsAdjacent = true + of lineEnd: break + of '"': + while true: + inc(peek) + case lex.at(T, peek) + of lineEnd, '"': break + of '\\': inc(peek) + else: discard + if lex.at(T, peek) != '"': break + recentAllowsAdjacent = true + of '\'': + inc(peek) + while lex.at(T, peek) notin {'\''} + lineEnd: inc(peek) + if lex.at(T, peek) != '\'': break + recentAllowsAdjacent = true + of '?', ',': + if flowDepth == 0: break + of '#': + if lex.at(T, peek - 1) in space: break + lex.skipPlainScalarContent() + continue + of '&', '*', '!': + inc(peek) + while lex.at(T, peek) notin spaceOrLineEnd: inc(peek) + recentAllowsAdjacent = false + continue + of space: discard + else: + lex.skipPlainScalarContent() + continue + inc(peek) + +# interface + +proc init*[T](lex: YamlLexer) = + lex.nextState = outsideDoc[T] + lex.lineStartState = outsideDoc[T] + lex.inlineState = insideLine[T] + lex.insideLineImpl = insideLine[T] + lex.insideDocImpl = insideDoc[T] + lex.insideFlowImpl = insideFlow[T] + lex.outsideDocImpl = outsideDoc[T] # only needed because of compiler checks + lex.tokenLineGetter = tokenLine[T] + lex.searchColonImpl = searchColon[T] + +proc newYamlLexer*(source: Stream): YamlLexer {.raises: [YamlLexerError].} = + let blSource = new(BaseLexer) + try: blSource[].open(source) + except: + var e = newException(YamlLexerError, + "Could not open stream for reading:\n" & getCurrentExceptionMsg()) + e.parent = getCurrentException() + raise e + GC_ref(blSource) + new(result, proc(x: ref YamlLexerObj) {.nimcall.} = + GC_unref(cast[ref BaseLexer](x.source)) + ) + result[] = YamlLexerObj(source: cast[pointer](blSource), inFlow: false, + buf: "", c: blSource[].buf[blSource[].bufpos], newlines: 0, + folded: true) + init[BaseLexer](result) + +proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer + {.raises: [].} = + let sSource = new(StringSource) + sSource[] = StringSource(pos: startAt, lineStart: startAt, line: 1, + src: source) + GC_ref(sSource) + new(result, proc(x: ref YamlLexerObj) {.nimcall.} = + GC_unref(cast[ref StringSource](x.source)) + ) + result[] = YamlLexerObj(buf: "", source: cast[pointer](sSource), + inFlow: false, c: sSource.src[startAt], newlines: 0, folded: true) + init[StringSource](result) + +proc next*(lex: YamlLexer) = + while not lex.nextState(lex): discard + debug("lexer -> " & $lex.cur) + +proc setFlow*(lex: YamlLexer, value: bool) = + lex.inFlow = value + # in flow mode, no indentation tokens are generated because they are not + # necessary. actually, the lexer will behave wrongly if we do that, because + # adjacent values need to check if the preceding token was a JSON value, and + # if indentation tokens are generated, that information is not available. + # therefore, we use insideFlow instead of insideDoc in flow mode. another + # reason is that this would erratically check for document markers (---, ...) + # which are simply scalars in flow mode. + if value: lex.lineStartState = lex.insideFlowImpl + else: lex.lineStartState = lex.insideDocImpl + +proc endBlockScalar*(lex: YamlLexer) = + lex.inlineState = lex.insideLineImpl + lex.nextState = lex.insideLineImpl + lex.folded = true + +proc getTokenLine*(lex: YamlLexer, marker: bool = true): string = + result = lex.tokenLineGetter(lex, lex.curStartPos, marker) + +proc getTokenLine*(lex: YamlLexer, pos: tuple[line, column: int], + marker: bool = true): string = + result = lex.tokenLineGetter(lex, pos, marker) + +proc isImplicitKeyStart*(lex: YamlLexer): bool = + result = lex.searchColonImpl(lex) diff --git a/private/serialization.nim b/private/serialization.nim deleted file mode 100644 index 491a4c7..0000000 --- a/private/serialization.nim +++ /dev/null @@ -1,883 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -proc newConstructionContext*(): ConstructionContext = - new(result) - result.refs = initTable[AnchorId, pointer]() - -proc newSerializationContext*(s: AnchorStyle): SerializationContext = - new(result) - result.refs = initTable[pointer, AnchorId]() - result.style = s - result.nextAnchorId = 0.AnchorId - -template presentTag*(t: typedesc, ts: TagStyle): TagId = - ## Get the TagId that represents the given type in the given style - if ts == tsNone: yTagQuestionMark else: yamlTag(t) - -proc lazyLoadTag(uri: string): TagId {.inline, raises: [].} = - try: result = serializationTagLibrary.tags[uri] - except KeyError: result = serializationTagLibrary.registerUri(uri) - -proc safeTagUri(id: TagId): string {.raises: [].} = - try: - let uri = serializationTagLibrary.uri(id) - if uri.len > 0 and uri[0] == '!': return uri[1..uri.len - 1] - else: return uri - except KeyError: - # cannot happen (theoretically, you know) - assert(false) - -template constructScalarItem*(s: var YamlStream, i: expr, - t: typedesc, content: untyped) = - ## Helper template for implementing ``constructObject`` for types that - ## are constructed from a scalar. ``i`` is the identifier that holds - ## the scalar as ``YamlStreamEvent`` in the content. Exceptions raised in - ## the content will be automatically catched and wrapped in - ## ``YamlConstructionError``, which will then be raised. - let i = s.next() - if i.kind != yamlScalar: - raise newException(YamlConstructionError, "Expected scalar") - try: content - except YamlConstructionError: raise - except Exception: - var e = newException(YamlConstructionError, - "Cannot construct to " & name(t) & ": " & item.scalarContent) - e.parent = getCurrentException() - raise e - -proc yamlTag*(T: typedesc[string]): TagId {.inline, noSideEffect, raises: [].} = - yTagString - -proc constructObject*(s: var YamlStream, c: ConstructionContext, - result: var string) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## costructs a string from a YAML scalar - constructScalarItem(s, item, string): - result = item.scalarContent - -proc representObject*(value: string, ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - ## represents a string as YAML scalar - result = iterator(): YamlStreamEvent = - yield scalarEvent(value, tag, yAnchorNone) - -proc constructObject*[T: int8|int16|int32|int64]( - s: var YamlStream, c: ConstructionContext, result: var T) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs an integer value from a YAML scalar - constructScalarItem(s, item, T): - result = T(parseBiggestInt(item.scalarContent)) - -proc constructObject*(s: var YamlStream, c: ConstructionContext, - result: var int) - {.raises: [YamlConstructionError, YamlStreamError], inline.} = - ## constructs an integer of architecture-defined length by loading it into - ## int32 and then converting it. - var i32Result: int32 - constructObject(s, c, i32Result) - result = int(i32Result) - -proc representObject*[T: int8|int16|int32|int64](value: T, ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - ## represents an integer value as YAML scalar - result = iterator(): YamlStreamEvent = - yield scalarEvent($value, tag, yAnchorNone) - -proc representObject*(value: int, tagStyle: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream - {.raises: [], inline.}= - ## represent an integer of architecture-defined length by casting it to int32. - ## on 64-bit systems, this may cause a type conversion error. - - # currently, sizeof(int) is at least sizeof(int32). - representObject(int32(value), tagStyle, c, tag) - -{.push overflowChecks: on.} -proc parseBiggestUInt(s: string): uint64 = - result = 0 - for c in s: - if c in {'0'..'9'}: result *= 10.uint64 + (uint64(c) - uint64('0')) - elif c == '_': discard - else: raise newException(ValueError, "Invalid char in uint: " & c) -{.pop.} - -proc constructObject*[T: uint8|uint16|uint32|uint64]( - s: var YamlStream, c: ConstructionContext, result: var T) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## construct an unsigned integer value from a YAML scalar - constructScalarItem(s, item, T): - result = T(parseBiggestUInt(item.scalarContent)) - -proc constructObject*(s: var YamlStream, c: ConstructionContext, - result: var uint) - {.raises: [YamlConstructionError, YamlStreamError], inline.} = - ## represent an unsigned integer of architecture-defined length by loading it - ## into uint32 and then converting it. - var u32Result: uint32 - constructObject(s, c, u32Result) - result= uint(u32Result) - -proc representObject*[T: uint8|uint16|uint32|uint64](value: T, ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - ## represents an unsigned integer value as YAML scalar - result = iterator(): YamlStreamEvent = - yield scalarEvent($value, tag, yAnchorNone) - -proc representObject*(value: uint, ts: TagStyle, c: SerializationContext, - tag: TagId): RawYamlStream {.raises: [], inline.} = - ## represent an unsigned integer of architecture-defined length by casting it - ## to int32. on 64-bit systems, this may cause a type conversion error. - representObject(uint32(value), ts, c, tag) - -proc constructObject*[T: float|float32|float64]( - s: var YamlStream, c: ConstructionContext, result: var T) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## construct a float value from a YAML scalar - constructScalarItem(s, item, T): - let hint = guessType(item.scalarContent) - case hint - of yTypeFloat: - discard parseBiggestFloat(item.scalarContent, result) - of yTypeInteger: - discard parseBiggestFloat(item.scalarContent, result) - of yTypeFloatInf: - if item.scalarContent[0] == '-': result = NegInf - else: result = Inf - of yTypeFloatNaN: result = NaN - else: - raise newException(YamlConstructionError, - "Cannot construct to float: " & item.scalarContent) - -proc representObject*[T: float|float32|float64](value: T, ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - ## represents a float value as YAML scalar - result = iterator(): YamlStreamEvent = - var asString: string - case value - of Inf: asString = ".inf" - of NegInf: asString = "-.inf" - of NaN: asString = ".nan" - else: asString = $value - yield scalarEvent(asString, tag, yAnchorNone) - -proc yamlTag*(T: typedesc[bool]): TagId {.inline, raises: [].} = yTagBoolean - -proc constructObject*(s: var YamlStream, c: ConstructionContext, - result: var bool) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a bool value from a YAML scalar - constructScalarItem(s, item, bool): - case guessType(item.scalarContent) - of yTypeBoolTrue: result = true - of yTypeBoolFalse: result = false - else: - raise newException(YamlConstructionError, - "Cannot construct to bool: " & item.scalarContent) - -proc representObject*(value: bool, ts: TagStyle, c: SerializationContext, - tag: TagId): RawYamlStream {.raises: [].} = - ## represents a bool value as a YAML scalar - result = iterator(): YamlStreamEvent = - yield scalarEvent(if value: "y" else: "n", tag, yAnchorNone) - -proc constructObject*(s: var YamlStream, c: ConstructionContext, - result: var char) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a char value from a YAML scalar - constructScalarItem(s, item, char): - if item.scalarContent.len != 1: - raise newException(YamlConstructionError, - "Cannot construct to char (length != 1): " & item.scalarContent) - else: result = item.scalarContent[0] - -proc representObject*(value: char, ts: TagStyle, c: SerializationContext, - tag: TagId): RawYamlStream {.raises: [].} = - ## represents a char value as YAML scalar - result = iterator(): YamlStreamEvent = - yield scalarEvent("" & value, tag, yAnchorNone) - -proc yamlTag*[I](T: typedesc[seq[I]]): TagId {.inline, raises: [].} = - let uri = "!nim:system:seq(" & safeTagUri(yamlTag(I)) & ')' - result = lazyLoadTag(uri) - -proc yamlTag*[I](T: typedesc[set[I]]): TagId {.inline, raises: [].} = - let uri = "!nim:system:set(" & safeTagUri(yamlTag(I)) & ')' - result = lazyLoadTag(uri) - -proc constructObject*[T](s: var YamlStream, c: ConstructionContext, - result: var seq[T]) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a Nim seq from a YAML sequence - let event = s.next() - if event.kind != yamlStartSeq: - raise newException(YamlConstructionError, "Expected sequence start") - result = newSeq[T]() - while s.peek().kind != yamlEndSeq: - var item: T - constructChild(s, c, item) - result.add(item) - discard s.next() - -proc constructObject*[T](s: var YamlStream, c: ConstructionContext, - result: var set[T]) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a Nim seq from a YAML sequence - let event = s.next() - if event.kind != yamlStartSeq: - raise newException(YamlConstructionError, "Expected sequence start") - result = {} - while s.peek().kind != yamlEndSeq: - var item: T - constructChild(s, c, item) - result.incl(item) - discard s.next() - -proc representObject*[T](value: seq[T]|set[T], ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - ## represents a Nim seq as YAML sequence - result = iterator(): YamlStreamEvent = - let childTagStyle = if ts == tsRootOnly: tsNone else: ts - yield startSeqEvent(tag) - for item in value: - var events = representChild(item, childTagStyle, c) - while true: - let event = events() - if finished(events): break - yield event - yield endSeqEvent() - -proc yamlTag*[I, V](T: typedesc[array[I, V]]): TagId {.inline, raises: [].} = - const rangeName = name(I) - let uri = "!nim:system:array(" & rangeName[6..rangeName.high()] & "," & - safeTagUri(yamlTag(V)) & ')' - result = lazyLoadTag(uri) - -proc constructObject*[I, T](s: var YamlStream, c: ConstructionContext, - result: var array[I, T]) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a Nim array from a YAML sequence - var event = s.next() - if event.kind != yamlStartSeq: - raise newException(YamlConstructionError, "Expected sequence start") - for index in low(I)..high(I): - event = s.peek() - if event.kind == yamlEndSeq: - raise newException(YamlConstructionError, "Too few array values") - constructChild(s, c, result[index]) - event = s.next() - if event.kind != yamlEndSeq: - raise newException(YamlConstructionError, "Too much array values") - -proc representObject*[I, T](value: array[I, T], ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - ## represents a Nim array as YAML sequence - result = iterator(): YamlStreamEvent = - let childTagStyle = if ts == tsRootOnly: tsNone else: ts - yield startSeqEvent(tag) - for item in value: - var events = representChild(item, childTagStyle, c) - while true: - let event = events() - if finished(events): break - yield event - yield endSeqEvent() - -proc yamlTag*[K, V](T: typedesc[Table[K, V]]): TagId {.inline, raises: [].} = - try: - let uri = "!nim:tables:Table(" & safeTagUri(yamlTag(K)) & "," & - safeTagUri(yamlTag(V)) & ")" - result = lazyLoadTag(uri) - except KeyError: - # cannot happen (theoretically, you know) - assert(false) - -proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext, - result: var Table[K, V]) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a Nim Table from a YAML mapping - let event = s.next() - if event.kind != yamlStartMap: - raise newException(YamlConstructionError, "Expected map start, got " & - $event.kind) - result = initTable[K, V]() - while s.peek.kind != yamlEndMap: - var - key: K - value: V - constructChild(s, c, key) - constructChild(s, c, value) - if result.contains(key): - raise newException(YamlConstructionError, "Duplicate table key!") - result[key] = value - discard s.next() - -proc representObject*[K, V](value: Table[K, V], ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises:[].} = - ## represents a Nim Table as YAML mapping - result = iterator(): YamlStreamEvent = - let childTagStyle = if ts == tsRootOnly: tsNone else: ts - yield startMapEvent(tag) - for key, value in value.pairs: - var events = representChild(key, childTagStyle, c) - while true: - let event = events() - if finished(events): break - yield event - events = representChild(value, childTagStyle, c) - while true: - let event = events() - if finished(events): break - yield event - yield endMapEvent() - -proc yamlTag*[K, V](T: typedesc[OrderedTable[K, V]]): TagId - {.inline, raises: [].} = - try: - let uri = "!nim:tables:OrderedTable(" & safeTagUri(yamlTag(K)) & "," & - safeTagUri(yamlTag(V)) & ")" - result = lazyLoadTag(uri) - except KeyError: - # cannot happen (theoretically, you know) - assert(false) - -proc constructObject*[K, V](s: var YamlStream, c: ConstructionContext, - result: var OrderedTable[K, V]) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a Nim OrderedTable from a YAML mapping - let event = s.next() - if event.kind != yamlStartSeq: - raise newException(YamlConstructionError, "Expected seq start, got " & - $event.kind) - result = initOrderedTable[K, V]() - while s.peek.kind != yamlEndSeq: - var - key: K - value: V - if s.next().kind != yamlStartMap: - raise newException(YamlConstructionError, - "Expected map start, got " & $event.kind) - constructChild(s, c, key) - constructChild(s, c, value) - if s.next().kind != yamlEndMap: - raise newException(YamlConstructionError, - "Expected map end, got " & $event.kind) - if result.contains(key): - raise newException(YamlConstructionError, "Duplicate table key!") - result.add(key, value) - discard s.next() - -proc representObject*[K, V](value: OrderedTable[K, V], ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - result = iterator(): YamlStreamEvent = - let childTagStyle = if ts == tsRootOnly: tsNone else: ts - yield startSeqEvent(tag) - for key, value in value.pairs: - yield startMapEvent() - var events = representChild(key, childTagStyle, c) - while true: - let event = events() - if finished(events): break - yield event - events = representChild(value, childTagStyle, c) - while true: - let event = events() - if finished(events): break - yield event - yield endMapEvent() - yield endSeqEvent() - -proc yamlTag*(T: typedesc[object|enum]): - TagId {.inline, raises: [].} = - var uri = "!nim:custom:" & (typetraits.name(type(T))) - try: serializationTagLibrary.tags[uri] - except KeyError: serializationTagLibrary.registerUri(uri) - -proc yamlTag*(T: typedesc[tuple]): - TagId {.inline, raises: [].} = - var - i: T - uri = "!nim:tuple(" - first = true - for name, value in fieldPairs(i): - if first: first = false - else: uri.add(",") - uri.add(safeTagUri(yamlTag(type(value)))) - uri.add(")") - try: serializationTagLibrary.tags[uri] - except KeyError: serializationTagLibrary.registerUri(uri) - -macro constructFieldValue(t: typedesc, stream: expr, context: expr, - name: expr, o: expr): stmt = - let tDesc = getType(getType(t)[1]) - result = newNimNode(nnkCaseStmt).add(name) - for child in tDesc[2].children: - if child.kind == nnkRecCase: - let - discriminant = newDotExpr(o, newIdentNode($child[0])) - discType = newCall("type", discriminant) - var disOb = newNimNode(nnkOfBranch).add(newStrLitNode($child[0])) - disOb.add(newStmtList( - newNimNode(nnkVarSection).add( - newNimNode(nnkIdentDefs).add( - newIdentNode("value"), discType, newEmptyNode())), - newCall("constructChild", stream, context, newIdentNode("value")), - newAssignment(discriminant, newIdentNode("value")))) - result.add(disOb) - for bIndex in 1 .. len(child) - 1: - let discTest = infix(discriminant, "==", child[bIndex][0]) - for item in child[bIndex][1].children: - assert item.kind == nnkSym - var ob = newNimNode(nnkOfBranch).add(newStrLitNode($item)) - let field = newDotExpr(o, newIdentNode($item)) - var ifStmt = newIfStmt((cond: discTest, body: newStmtList( - newCall("constructChild", stream, context, field)))) - ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkRaiseStmt).add( - newCall("newException", newIdentNode("YamlConstructionError"), - infix(newStrLitNode("Field " & $item & " not allowed for " & - $child[0] & " == "), "&", prefix(discriminant, "$")))))) - ob.add(newStmtList(ifStmt)) - result.add(ob) - else: - assert child.kind == nnkSym - var ob = newNimNode(nnkOfBranch).add(newStrLitNode($child)) - let field = newDotExpr(o, newIdentNode($child)) - ob.add(newStmtList(newCall("constructChild", stream, context, field))) - result.add(ob) - -proc isVariantObject(t: typedesc): bool {.compileTime.} = - let tDesc = getType(t) - if tDesc.kind != nnkObjectTy: return false - for child in tDesc[2].children: - if child.kind == nnkRecCase: return true - return false - -proc constructObject*[O: object|tuple]( - s: var YamlStream, c: ConstructionContext, result: var O) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a Nim object or tuple from a YAML mapping - let e = s.next() - const - startKind = when isVariantObject(O): yamlStartSeq else: yamlStartMap - endKind = when isVariantObject(O): yamlEndSeq else: yamlEndMap - if e.kind != startKind: - raise newException(YamlConstructionError, "While constructing " & - typetraits.name(O) & ": Expected map start, got " & $e.kind) - when isVariantObject(O): reset(result) # make discriminants writeable - while s.peek.kind != endKind: - # todo: check for duplicates in input and raise appropriate exception - # also todo: check for missing items and raise appropriate exception - var e = s.next() - when isVariantObject(O): - if e.kind != yamlStartMap: - raise newException(YamlConstructionError, - "Expected single-pair map, got " & $e.kind) - e = s.next() - if e.kind != yamlScalar: - raise newException(YamlConstructionError, - "Expected field name, got " & $e.kind) - let name = e.scalarContent - when result is tuple: - for fname, value in fieldPairs(result): - if fname == name: - constructChild(s, c, value) - break - else: - constructFieldValue(O, s, c, name, result) - when isVariantObject(O): - e = s.next() - if e.kind != yamlEndMap: - raise newException(YamlConstructionError, - "Expected end of single-pair map, got " & $e.kind) - discard s.next() - -proc representObject*[O: object|tuple](value: O, ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - ## represents a Nim object or tuple as YAML mapping - result = iterator(): YamlStreamEvent = - let childTagStyle = if ts == tsRootOnly: tsNone else: ts - when isVariantObject(O): - yield startSeqEvent(tag, yAnchorNone) - else: - yield startMapEvent(tag, yAnchorNone) - for name, value in fieldPairs(value): - when isVariantObject(O): - yield startMapEvent(yTagQuestionMark, yAnchorNone) - yield scalarEvent(name, - if childTagStyle == tsNone: yTagQuestionMark else: - yTagNimField, yAnchorNone) - var events = representChild(value, childTagStyle, c) - while true: - let event = events() - if finished(events): break - yield event - when isVariantObject(O): - yield endMapEvent() - when isVariantObject(O): - yield endSeqEvent() - else: - yield endMapEvent() - -proc constructObject*[O: enum](s: var YamlStream, c: ConstructionContext, - result: var O) - {.raises: [YamlConstructionError, YamlStreamError].} = - ## constructs a Nim enum from a YAML scalar - let e = s.next() - if e.kind != yamlScalar: - raise newException(YamlConstructionError, "Expected scalar, got " & - $e.kind) - try: result = parseEnum[O](e.scalarContent) - except ValueError: - var ex = newException(YamlConstructionError, "Cannot parse '" & - e.scalarContent & "' as " & type(O).name) - ex.parent = getCurrentException() - raise ex - -proc representObject*[O: enum](value: O, ts: TagStyle, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - ## represents a Nim enum as YAML scalar - result = iterator(): YamlStreamEvent = - yield scalarEvent($value, tag, yAnchorNone) - -proc yamlTag*[O](T: typedesc[ref O]): TagId {.inline, raises: [].} = yamlTag(O) - -macro constructImplicitVariantObject(s, c, r, possibleTagIds: expr, - t: typedesc): stmt = - let tDesc = getType(getType(t)[1]) - assert tDesc.kind == nnkObjectTy - let recCase = tDesc[2][0] - assert recCase.kind == nnkRecCase - let - discriminant = newDotExpr(r, newIdentNode($recCase[0])) - discType = newCall("type", discriminant) - var ifStmt = newNimNode(nnkIfStmt) - for i in 1 .. recCase.len - 1: - assert recCase[i].kind == nnkOfBranch - var branch = newNimNode(nnkElifBranch) - var branchContent = newStmtList(newAssignment(discriminant, recCase[i][0])) - case recCase[i][1].len - of 0: - branch.add(infix(newIdentNode("yTagNull"), "in", possibleTagIds)) - branchContent.add(newNimNode(nnkDiscardStmt).add(newCall("next", s))) - of 1: - let field = newDotExpr(r, newIdentNode($recCase[i][1][0])) - branch.add(infix( - newCall("yamlTag", newCall("type", field)), "in", possibleTagIds)) - branchContent.add(newCall("constructChild", s, c, field)) - else: assert false - branch.add(branchContent) - ifStmt.add(branch) - let raiseStmt = newNimNode(nnkRaiseStmt).add( - newCall("newException", newIdentNode("YamlConstructionError"), - infix(newStrLitNode("This value type does not map to any field in " & - typetraits.name(t) & ": "), "&", - newCall("uri", newIdentNode("serializationTagLibrary"), - newNimNode(nnkBracketExpr).add(possibleTagIds, newIntLitNode(0))) - ) - )) - ifStmt.add(newNimNode(nnkElse).add(newNimNode(nnkTryStmt).add( - newStmtList(raiseStmt), newNimNode(nnkExceptBranch).add( - newIdentNode("KeyError"), newStmtList(newCall("assert", newLit(false))) - )))) - result = newStmtList(newCall("reset", r), ifStmt) - -proc constructChild*[T](s: var YamlStream, c: ConstructionContext, - result: var T) = - let item = s.peek() - when compiles(implicitVariantObject(result)): - var possibleTagIds = newSeq[TagId]() - case item.kind - of yamlScalar: - case item.scalarTag - of yTagQuestionMark: - case guessType(item.scalarContent) - of yTypeInteger: - possibleTagIds.add([yamlTag(int), yamlTag(int8), yamlTag(int16), - yamlTag(int32), yamlTag(int64)]) - if item.scalarContent[0] != '-': - possibleTagIds.add([yamlTag(uint), yamlTag(uint8), yamlTag(uint16), - yamlTag(uint32), yamlTag(uint64)]) - of yTypeFloat, yTypeFloatInf, yTypeFloatNaN: - possibleTagIds.add([yamlTag(float), yamlTag(float32), - yamlTag(float64)]) - of yTypeBoolTrue, yTypeBoolFalse: - possibleTagIds.add(yamlTag(bool)) - of yTypeNull: - raise newException(YamlConstructionError, "not implemented!") - of yTypeUnknown: - possibleTagIds.add(yamlTag(string)) - of yTagExclamationMark: - possibleTagIds.add(yamlTag(string)) - else: - possibleTagIds.add(item.scalarTag) - of yamlStartMap: - if item.mapTag in [yTagQuestionMark, yTagExclamationMark]: - raise newException(YamlConstructionError, - "Complex value of implicit variant object type must have a tag.") - possibleTagIds.add(item.mapTag) - of yamlStartSeq: - if item.seqTag in [yTagQuestionMark, yTagExclamationMark]: - raise newException(YamlConstructionError, - "Complex value of implicit variant object type must have a tag.") - possibleTagIds.add(item.seqTag) - else: assert false - constructImplicitVariantObject(s, c, result, possibleTagIds, T) - else: - case item.kind - of yamlScalar: - if item.scalarTag notin [yTagQuestionMark, yTagExclamationMark, - yamlTag(T)]: - raise newException(YamlConstructionError, "Wrong tag for " & - typetraits.name(T)) - elif item.scalarAnchor != yAnchorNone: - raise newException(YamlConstructionError, "Anchor on non-ref type") - of yamlStartMap: - if item.mapTag notin [yTagQuestionMark, yamlTag(T)]: - raise newException(YamlConstructionError, "Wrong tag for " & - typetraits.name(T)) - elif item.mapAnchor != yAnchorNone: - raise newException(YamlConstructionError, "Anchor on non-ref type") - of yamlStartSeq: - if item.seqTag notin [yTagQuestionMark, yamlTag(T)]: - raise newException(YamlConstructionError, "Wrong tag for " & - typetraits.name(T)) - elif item.seqAnchor != yAnchorNone: - raise newException(YamlConstructionError, "Anchor on non-ref type") - else: assert false - constructObject(s, c, result) - -proc constructChild*(s: var YamlStream, c: ConstructionContext, - result: var string) = - let item = s.peek() - if item.kind == yamlScalar: - if item.scalarTag == yTagNimNilString: - discard s.next() - result = nil - return - elif item.scalarTag notin - [yTagQuestionMark, yTagExclamationMark, yamlTag(string)]: - raise newException(YamlConstructionError, "Wrong tag for string") - elif item.scalarAnchor != yAnchorNone: - raise newException(YamlConstructionError, "Anchor on non-ref type") - constructObject(s, c, result) - -proc constructChild*[T](s: var YamlStream, c: ConstructionContext, - result: var seq[T]) = - let item = s.peek() - if item.kind == yamlScalar: - if item.scalarTag == yTagNimNilSeq: - discard s.next() - result = nil - return - elif item.kind == yamlStartSeq: - if item.seqTag notin [yTagQuestionMark, yamlTag(seq[T])]: - raise newException(YamlConstructionError, "Wrong tag for " & - typetraits.name(seq[T])) - elif item.seqAnchor != yAnchorNone: - raise newException(YamlConstructionError, "Anchor on non-ref type") - constructObject(s, c, result) - -proc constructChild*[O](s: var YamlStream, c: ConstructionContext, - result: var ref O) = - var e = s.peek() - if e.kind == yamlScalar: - if e.scalarTag == yTagNull or (e.scalarTag == yTagQuestionMark and - guessType(e.scalarContent) == yTypeNull): - result = nil - discard s.next() - return - elif e.kind == yamlAlias: - try: - result = cast[ref O](c.refs[e.aliasTarget]) - discard s.next() - return - except KeyError: assert(false) - new(result) - template removeAnchor(anchor: var AnchorId) {.dirty.} = - if anchor != yAnchorNone: - assert(not c.refs.hasKey(anchor)) - c.refs[anchor] = cast[pointer](result) - anchor = yAnchorNone - - case e.kind - of yamlScalar: removeAnchor(e.scalarAnchor) - of yamlStartMap: removeAnchor(e.mapAnchor) - of yamlStartSeq: removeAnchor(e.seqAnchor) - else: assert(false) - s.peek = e - try: constructChild(s, c, result[]) - except YamlConstructionError, YamlStreamError, AssertionError: raise - except Exception: - var e = newException(YamlStreamError, getCurrentExceptionMsg()) - e.parent = getCurrentException() - raise e - -proc representChild*(value: string, ts: TagStyle, c: SerializationContext): - RawYamlStream = - if isNil(value): - result = iterator(): YamlStreamEvent = - yield scalarEvent("", yTagNimNilString) - else: result = representObject(value, ts, c, presentTag(string, ts)) - -proc representChild*[T](value: seq[T], ts: TagStyle, c: SerializationContext): - RawYamlStream = - if isNil(value): - result = iterator(): YamlStreamEvent = - yield scalarEvent("", yTagNimNilSeq) - else: result = representObject(value, ts, c, presentTag(seq[T], ts)) - -proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext): - RawYamlStream = - if isNil(value): - result = iterator(): YamlStreamEvent = - yield scalarEvent("~", yTagNull) - elif c.style == asNone: result = representChild(value[], ts, c) - else: - let p = cast[pointer](value) - if c.refs.hasKey(p): - try: - if c.refs[p] == yAnchorNone: - c.refs[p] = c.nextAnchorId - c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1) - except KeyError: assert false, "Can never happen" - result = iterator(): YamlStreamEvent {.raises: [].} = - var event: YamlStreamEvent - try: event = aliasEvent(c.refs[p]) - except KeyError: assert false, "Can never happen" - yield event - return - try: - if c.style == asAlways: - c.refs[p] = c.nextAnchorId - c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1) - else: c.refs[p] = yAnchorNone - let - a = if c.style == asAlways: c.refs[p] else: cast[AnchorId](p) - childTagStyle = if ts == tsAll: tsAll else: tsRootOnly - result = iterator(): YamlStreamEvent = - var child = representChild(value[], childTagStyle, c) - var first = child() - assert(not finished(child)) - case first.kind - of yamlStartMap: - first.mapAnchor = a - if ts == tsNone: first.mapTag = yTagQuestionMark - of yamlStartSeq: - first.seqAnchor = a - if ts == tsNone: first.seqTag = yTagQuestionMark - of yamlScalar: - first.scalarAnchor = a - if ts == tsNone and guessType(first.scalarContent) != yTypeNull: - first.scalarTag = yTagQuestionMark - else: discard - yield first - while true: - let event = child() - if finished(child): break - yield event - except KeyError: assert false, "Can never happen" - -proc representChild*[O](value: O, ts: TagStyle, - c: SerializationContext): RawYamlStream = - when compiles(implicitVariantObject(value)): - # todo: this would probably be nicer if constructed with a macro - var count = 0 - for name, field in fieldPairs(value): - if count > 0: - result = - representChild(field, if ts == tsAll: tsAll else: tsRootOnly, c) - inc(count) - if count == 1: - result = iterator(): YamlStreamEvent = - yield scalarEvent("~", yTagNull) - else: - result = representObject(value, ts, c, if ts == tsNone: - yTagQuestionMark else: yamlTag(O)) - -proc construct*[T](s: var YamlStream, target: var T) = - var context = newConstructionContext() - try: - var e = s.next() - assert(e.kind == yamlStartDoc) - - constructChild(s, context, target) - e = s.next() - assert(e.kind == yamlEndDoc) - except YamlConstructionError: - raise (ref YamlConstructionError)(getCurrentException()) - except YamlStreamError: - raise (ref YamlStreamError)(getCurrentException()) - except AssertionError: - raise (ref AssertionError)(getCurrentException()) - except Exception: - # may occur while calling s() - var ex = newException(YamlStreamError, "") - ex.parent = getCurrentException() - raise ex - -proc load*[K](input: Stream, target: var K) = - var - parser = newYamlParser(serializationTagLibrary) - events = parser.parse(input) - try: construct(events, target) - except YamlConstructionError: - var e = (ref YamlConstructionError)(getCurrentException()) - e.line = parser.getLineNumber() - e.column = parser.getColNumber() - e.lineContent = parser.getLineContent() - raise e - except YamlStreamError: - let e = (ref YamlStreamError)(getCurrentException()) - if e.parent of IOError: raise (ref IOError)(e.parent) - elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent) - else: assert false - -proc setAnchor(a: var AnchorId, q: var Table[pointer, AnchorId]) - {.inline.} = - if a != yAnchorNone: - try: a = q[cast[pointer](a)] - except KeyError: assert false, "Can never happen" - -proc represent*[T](value: T, ts: TagStyle = tsRootOnly, - a: AnchorStyle = asTidy): YamlStream = - var - context = newSerializationContext(a) - objStream = iterator(): YamlStreamEvent = - yield startDocEvent() - var events = representChild(value, ts, context) - while true: - let e = events() - if finished(events): break - yield e - yield endDocEvent() - if a == asTidy: - var objQueue = newSeq[YamlStreamEvent]() - try: - for event in objStream(): objQueue.add(event) - except Exception: - assert(false) - var backend = iterator(): YamlStreamEvent = - for i in countup(0, objQueue.len - 1): - var event = objQueue[i] - case event.kind - of yamlStartMap: event.mapAnchor.setAnchor(context.refs) - of yamlStartSeq: event.seqAnchor.setAnchor(context.refs) - of yamlScalar: event.scalarAnchor.setAnchor(context.refs) - else: discard - yield event - result = initYamlStream(backend) - else: result = initYamlStream(objStream) - -proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly, - anchorStyle: AnchorStyle = asTidy, - options: PresentationOptions = defaultPresentationOptions) = - var events = represent(value, - if options.style == psCanonical: tsAll else: tagStyle, - if options.style == psJson: asNone else: anchorStyle) - try: present(events, target, serializationTagLibrary, options) - except YamlStreamError: - # serializing object does not raise any errors, so we can ignore this - assert false, "Can never happen" diff --git a/private/streams.nim b/private/streams.nim deleted file mode 100644 index 24f1243..0000000 --- a/private/streams.nim +++ /dev/null @@ -1,61 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -proc initYamlStream*(backend: iterator(): YamlStreamEvent): YamlStream = - result.peeked = false - result.backend = backend - -proc next*(s: var YamlStream): YamlStreamEvent = - if s.peeked: - s.peeked = false - shallowCopy(result, s.cached) - else: - try: - shallowCopy(result, s.backend()) - assert(not finished(s.backend)) - except AssertionError: raise - except YamlStreamError: - let cur = getCurrentException() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur.parent - raise e - except Exception: - let cur = getCurrentException() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur - raise e - -proc peek*(s: var YamlStream): YamlStreamEvent = - if not s.peeked: - s.cached = s.next() - s.peeked = true - shallowCopy(result, s.cached) - -proc `peek=`*(s: var YamlStream, value: YamlStreamEvent) = - s.cached = value - s.peeked = true - -proc finished*(s: var YamlStream): bool = - if s.peeked: result = false - else: - try: - s.cached = s.backend() - if finished(s.backend): result = true - else: - s.peeked = true - result = false - except AssertionError: raise - except YamlStreamError: - let cur = getCurrentException() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur.parent - raise e - except Exception: - let cur = getCurrentException() - echo cur.getStackTrace() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur - raise e \ No newline at end of file diff --git a/private/tagLibrary.nim b/private/tagLibrary.nim deleted file mode 100644 index b6545fd..0000000 --- a/private/tagLibrary.nim +++ /dev/null @@ -1,83 +0,0 @@ -# NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. - -proc `$`*(id: TagId): string = - case id - of yTagQuestionMark: "?" - of yTagExclamationMark: "!" - of yTagString: "!!str" - of yTagSequence: "!!seq" - of yTagMapping: "!!map" - of yTagNull: "!!null" - of yTagBoolean: "!!bool" - of yTagInteger: "!!int" - of yTagFloat: "!!float" - of yTagOrderedMap: "!!omap" - of yTagPairs: "!!pairs" - of yTagSet: "!!set" - of yTagBinary: "!!binary" - of yTagMerge: "!!merge" - of yTagTimestamp: "!!timestamp" - of yTagValue: "!!value" - of yTagYaml: "!!yaml" - of yTagNimField: "!nim:field" - else: "<" & $cast[int](id) & ">" - -proc initTagLibrary*(): TagLibrary = - new(result) - result.tags = initTable[string, TagId]() - result.secondaryPrefix = yamlTagRepositoryPrefix - result.nextCustomTagId = yFirstCustomTagId - -proc registerUri*(tagLib: TagLibrary, uri: string): TagId = - tagLib.tags[uri] = tagLib.nextCustomTagId - result = tagLib.nextCustomTagId - tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1) - -proc uri*(tagLib: TagLibrary, id: TagId): string = - for iUri, iId in tagLib.tags.pairs: - if iId == id: return iUri - raise newException(KeyError, "Unknown tag id: " & $id) - -proc initFailsafeTagLibrary(): TagLibrary = - result = initTagLibrary() - result.tags["!"] = yTagExclamationMark - result.tags["?"] = yTagQuestionMark - result.tags["tag:yaml.org,2002:str"] = yTagString - result.tags["tag:yaml.org,2002:seq"] = yTagSequence - result.tags["tag:yaml.org,2002:map"] = yTagMapping - -proc initCoreTagLibrary(): TagLibrary = - result = initFailsafeTagLibrary() - result.tags["tag:yaml.org,2002:null"] = yTagNull - result.tags["tag:yaml.org,2002:bool"] = yTagBoolean - result.tags["tag:yaml.org,2002:int"] = yTagInteger - result.tags["tag:yaml.org,2002:float"] = yTagFloat - -proc initExtendedTagLibrary(): TagLibrary = - result = initCoreTagLibrary() - result.tags["tag:yaml.org,2002:omap"] = yTagOrderedMap - result.tags["tag:yaml.org,2002:pairs"] = yTagPairs - result.tags["tag:yaml.org,2002:binary"] = yTagBinary - result.tags["tag:yaml.org,2002:merge"] = yTagMerge - result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp - result.tags["tag:yaml.org,2002:value"] = yTagValue - result.tags["tag:yaml.org,2002:yaml"] = yTagYaml - -proc initSerializationTagLibrary(): TagLibrary = - result = initTagLibrary() - result.tags["!"] = yTagExclamationMark - result.tags["?"] = yTagQuestionMark - result.tags["tag:yaml.org,2002:str"] = yTagString - result.tags["tag:yaml.org,2002:null"] = yTagNull - result.tags["tag:yaml.org,2002:bool"] = yTagBoolean - result.tags["tag:yaml.org,2002:float"] = yTagFloat - result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp - result.tags["tag:yaml.org,2002:value"] = yTagValue - result.tags["tag:yaml.org,2002:binary"] = yTagBinary - result.tags["!nim:field"] = yTagNimField - result.tags["!nim:nil:string"] = yTagNimNilString - result.tags["!nim:nil:seq"] = yTagNimNilSeq \ No newline at end of file diff --git a/server/server.nim b/server/server.nim index 335006d..33edd10 100644 --- a/server/server.nim +++ b/server/server.nim @@ -6,7 +6,7 @@ import jester, asyncdispatch, json, streams, strutils import packages.docutils.rstgen, packages.docutils.highlite -import yaml +import ../yaml routes: get "/": @@ -16,7 +16,9 @@ routes: var style: PresentationStyle resultNode = newJObject() - tokens = false + msg: string + retStatus = Http200 + contentType = "application/json" headers["Access-Control-Allow-Origin"] = "*" headers["Pragma"] = "no-cache" headers["Cache-Control"] = "no-cache" @@ -36,14 +38,17 @@ routes: for event in events: output.add($event & "\n") resultNode["code"] = %0 resultNode["output"] = %output - resp resultNode.pretty, "application/json" - tokens = true - if not tokens: + msg = resultNode.pretty + else: + retStatus = Http400 + msg = "Invalid style: " & escape(@"style") + contentType = "text/plain;charset=utf8" + if isNil(msg): var output = newStringStream() highlighted = "" transform(newStringStream(@"input"), output, defineOptions(style)) - + # syntax highlighting (stolen and modified from stlib's rstgen) var g: GeneralTokenizer g.initGeneralTokenizer(output.data) @@ -60,7 +65,7 @@ routes: resultNode["code"] = %0 resultNode["output"] = %highlighted - resp resultNode.pretty, "application/json" + msg = resultNode.pretty except YamlParserError: let e = (ref YamlParserError)(getCurrentException()) resultNode["code"] = %1 @@ -68,17 +73,17 @@ routes: resultNode["column"] = %e.column resultNode["message"] = %e.msg resultNode["detail"] = %e.lineContent - resp resultNode.pretty, "application/json" + msg = resultNode.pretty except YamlPresenterJsonError: let e = getCurrentException() resultNode["code"] = %2 resultNode["message"] = %e.msg - headers["Content-Type"] = "application/json" - resp resultNode.pretty, "application/json" + msg = resultNode.pretty except: let e = getCurrentException() - let msg = "Name: " & $e.name & "\nMessage: " & e.msg & - "\nTrace:\n" & e.getStackTrace - resp Http500, msg, "text/plain;charset=utf-8" - + msg = "Name: " & $e.name & "\nMessage: " & e.msg & + "\nTrace:\n" & e.getStackTrace + retStatus = Http500 + contentType = "text/plain;charset=utf-8" + resp retStatus, msg, contentType runForever() diff --git a/test/common.nim b/test/commonTestUtils.nim similarity index 100% rename from test/common.nim rename to test/commonTestUtils.nim diff --git a/test/dom.nim b/test/tdom.nim similarity index 90% rename from test/dom.nim rename to test/tdom.nim index 6cc75af..2aa5c33 100644 --- a/test/dom.nim +++ b/test/tdom.nim @@ -5,21 +5,21 @@ # distribution, for details about the copyright. import "../yaml" -import unittest, common +import unittest, commonTestUtils, streams suite "DOM": - test "DOM: Composing simple Scalar": + test "Composing simple Scalar": let input = newStringStream("scalar") result = loadDOM(input) assert result.root.kind == yScalar assert result.root.content == "scalar" assert result.root.tag == "?" - test "DOM: Serializing simple Scalar": + test "Serializing simple Scalar": let input = initYamlDoc(newYamlNode("scalar")) var result = serialize(input, initExtendedTagLibrary()) ensure(result, startDocEvent(), scalarEvent("scalar"), endDocEvent()) - test "DOM: Composing sequence": + test "Composing sequence": let input = newStringStream("- !!str a\n- !!bool no") result = loadDOM(input) @@ -32,7 +32,7 @@ suite "DOM": assert result.root.children[1].kind == yScalar assert result.root.children[1].tag == "tag:yaml.org,2002:bool" assert result.root.children[1].content == "no" - test "DOM: Serializing sequence": + test "Serializing sequence": let input = initYamlDoc(newYamlNode([ newYamlNode("a", "tag:yaml.org,2002:str"), newYamlNode("no", "tag:yaml.org,2002:bool")])) @@ -40,7 +40,7 @@ suite "DOM": ensure(result, startDocEvent(), startSeqEvent(), scalarEvent("a", yTagString), scalarEvent("no", yTagBoolean), endSeqEvent(), endDocEvent()) - test "DOM: Composing mapping": + test "Composing mapping": let input = newStringStream("--- !!map\n!foo bar: [a, b]") result = loadDOM(input) @@ -52,7 +52,7 @@ suite "DOM": assert result.root.pairs[0].key.content == "bar" assert result.root.pairs[0].value.kind == ySequence assert result.root.pairs[0].value.children.len == 2 - test "DOM: Serializing mapping": + test "Serializing mapping": let input = initYamlDoc(newYamlNode([ (key: newYamlNode("bar"), value: newYamlNode([newYamlNode("a"), newYamlNode("b")]))])) @@ -60,7 +60,7 @@ suite "DOM": ensure(result, startDocEvent(), startMapEvent(), scalarEvent("bar"), startSeqEvent(), scalarEvent("a"), scalarEvent("b"), endSeqEvent(), endMapEvent(), endDocEvent()) - test "DOM: Composing with anchors": + test "Composing with anchors": let input = newStringStream("- &a foo\n- &b bar\n- *a\n- *b") result = loadDOM(input) @@ -72,7 +72,7 @@ suite "DOM": assert result.root.children[1].content == "bar" assert result.root.children[0] == result.root.children[2] assert result.root.children[1] == result.root.children[3] - test "DOM: Serializing with anchors": + test "Serializing with anchors": let a = newYamlNode("a") b = newYamlNode("b") @@ -83,7 +83,7 @@ suite "DOM": scalarEvent("b", anchor=1.AnchorId), scalarEvent("c"), aliasEvent(0.AnchorId), aliasEvent(1.AnchorId), endSeqEvent(), endDocEvent()) - test "DOM: Serializing with all anchors": + test "Serializing with all anchors": let a = newYamlNode("a") input = initYamlDoc(newYamlNode([a, newYamlNode("b"), a])) diff --git a/test/testEventParser.nim b/test/testEventParser.nim index 89796ce..f99e4a6 100644 --- a/test/testEventParser.nim +++ b/test/testEventParser.nim @@ -5,19 +5,19 @@ # distribution, for details about the copyright. import "../yaml" -import lexbase +import lexbase, streams, tables type LexerToken = enum plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, minusSeq, eqVal, eqAli, chevTag, andAnchor, quotContent, colonContent, noToken - + StreamPos = enum beforeStream, inStream, afterStream EventLexer = object of BaseLexer content: string - + EventStreamError = object of Exception proc nextToken(lex: var EventLexer): LexerToken = @@ -156,7 +156,7 @@ template setCurAnchor(val: AnchorId) = of yamlAlias: curEvent.aliasTarget = val else: raise newException(EventStreamError, $curEvent.kind & " may not have an anchor") - + template eventStart(k: YamlStreamEventKind) {.dirty.} = assertInStream() yieldEvent() diff --git a/test/tests.nim b/test/tests.nim index 74c9d28..26231ad 100644 --- a/test/tests.nim +++ b/test/tests.nim @@ -4,4 +4,4 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. -import constructingJson, serializing, dom \ No newline at end of file +import tlex, tjson, tserialization, tdom \ No newline at end of file diff --git a/test/constructingJson.nim b/test/tjson.nim similarity index 55% rename from test/constructingJson.nim rename to test/tjson.nim index 281ea43..4045766 100644 --- a/test/constructingJson.nim +++ b/test/tjson.nim @@ -6,28 +6,36 @@ import "../yaml" -import unittest, json +import unittest, json, streams proc wc(line, column: int, lineContent: string, message: string) = echo "Warning (", line, ",", column, "): ", message, "\n", lineContent proc ensureEqual(yamlIn, jsonIn: string) = - var - parser = newYamlParser(initCoreTagLibrary(), wc) - s = parser.parse(newStringStream(yamlIn)) - yamlResult = constructJson(s) - jsonResult = parseJson(jsonIn) - assert yamlResult.len == 1 - assert(jsonResult == yamlResult[0]) + try: + var + parser = newYamlParser(initCoreTagLibrary(), wc) + s = parser.parse(newStringStream(yamlIn)) + yamlResult = constructJson(s) + jsonResult = parseJson(jsonIn) + assert yamlResult.len == 1 + assert(jsonResult == yamlResult[0], "Expected: " & $jsonResult & ", got: " & + $yamlResult[0]) + except YamlStreamError: + let e = (ref YamlParserError)(getCurrentException().parent) + echo "error occurred: " & e.msg + echo "line: ", e.line, ", column: ", e.column + echo e.lineContent + raise e suite "Constructing JSON": - test "Constructing JSON: Simple Sequence": + test "Simple Sequence": ensureEqual("- 1\n- 2\n- 3", "[1, 2, 3]") - - test "Constructing JSON: Simple Map": + + test "Simple Map": ensureEqual("a: b\nc: d", """{"a": "b", "c": "d"}""") - - test "Constructing JSON: Complex Structure": + + test "Complex Structure": ensureEqual(""" %YAML 1.2 --- diff --git a/test/tlex.nim b/test/tlex.nim new file mode 100644 index 0000000..d6da0f1 --- /dev/null +++ b/test/tlex.nim @@ -0,0 +1,262 @@ +import ../private/lex + +import unittest, strutils + +const tokensWithValue = + {ltScalarPart, ltQuotedScalar, ltYamlVersion, ltTagShorthand, ltTagUri, + ltUnknownDirective, ltUnknownDirectiveParams, ltLiteralTag, ltAnchor, + ltAlias, ltBlockScalar} + +type + TokenWithValue = object + case kind: LexerToken + of tokensWithValue: + value: string + of ltIndentation: + indentation: int + of ltTagHandle: + handle, suffix: string + else: discard + +proc actualRepr(lex: YamlLexer, t: LexerToken): string = + result = $t + case t + of tokensWithValue + {ltTagHandle}: + result.add("(" & escape(lex.buf) & ")") + of ltIndentation: + result.add("(" & $lex.indentation & ")") + else: discard + +proc assertEquals(input: string, expected: varargs[TokenWithValue]) = + let lex = newYamlLexer(input) + var + i = 0 + blockScalarEnd = -1 + flowDepth = 0 + for expectedToken in expected: + inc(i) + try: + lex.next() + doAssert lex.cur == expectedToken.kind, "Wrong token kind at #" & $i & + ": Expected " & $expectedToken.kind & ", got " & + lex.actualRepr(lex.cur) + case expectedToken.kind + of tokensWithValue: + doAssert lex.buf == expectedToken.value, "Wrong token content at #" & + $i & ": Expected " & escape(expectedToken.value) & + ", got " & escape(lex.buf) + lex.buf = "" + of ltIndentation: + doAssert lex.indentation == expectedToken.indentation, + "Wrong indentation length at #" & $i & ": Expected " & + $expectedToken.indentation & ", got " & $lex.indentation + if lex.indentation <= blockScalarEnd: + lex.endBlockScalar() + blockScalarEnd = -1 + of ltBraceOpen, ltBracketOpen: + inc(flowDepth) + if flowDepth == 1: lex.setFlow(true) + of ltBraceClose, ltBracketClose: + dec(flowDepth) + if flowDepth == 0: lex.setFlow(false) + of ltTagHandle: + let + handle = lex.buf.substr(0, lex.shorthandEnd) + suffix = lex.buf.substr(lex.shorthandEnd + 1) + doAssert handle == expectedToken.handle, + "Wrong handle at #" & $i & ": Expected " & expectedToken.handle & + ", got " & handle + doAssert suffix == expectedToken.suffix, + "Wrong suffix at #" & $i & ": Expected " & expectedToken.suffix & + ", got " & suffix + lex.buf = "" + else: discard + except YamlLexerError: + let e = (ref YamlLexerError)(getCurrentException()) + echo "Error at line " & $e.line & ", column " & $e.column & ":" + echo e.lineContent + assert false + +proc assertLookahead(input: string, expected: bool, tokensBefore: int = 1) = + let lex = newYamlLexer(input) + var flowDepth = 0 + for i in 0..tokensBefore: + lex.next() + case lex.cur + of ltBraceOpen, ltBracketOpen: + inc(flowDepth) + if flowDepth == 1: lex.setFlow(true) + of ltBraceClose, ltBracketClose: + dec(flowDepth) + if flowDepth == 0: lex.setFlow(false) + else: discard + doAssert lex.isImplicitKeyStart() == expected + +proc i(indent: int): TokenWithValue = + TokenWithValue(kind: ltIndentation, indentation: indent) +proc sp(v: string): TokenWithValue = + TokenWithValue(kind: ltScalarPart, value: v) +proc qs(v: string): TokenWithValue = + TokenWithValue(kind: ltQuotedScalar, value: v) +proc se(): TokenWithValue = TokenWithValue(kind: ltStreamEnd) +proc mk(): TokenWithValue = TokenWithValue(kind: ltMapKeyInd) +proc mv(): TokenWithValue = TokenWithValue(kind: ltMapValInd) +proc si(): TokenWithValue = TokenWithValue(kind: ltSeqItemInd) +proc dy(): TokenWithValue = TokenWithValue(kind: ltYamlDirective) +proc dt(): TokenWithValue = TokenWithValue(kind: ltTagDirective) +proc du(v: string): TokenWithValue = + TokenWithValue(kind: ltUnknownDirective, value: v) +proc dp(v: string): TokenWithValue = + TokenWithValue(kind: ltUnknownDirectiveParams, value: v) +proc yv(v: string): TokenWithValue = + TokenWithValue(kind: ltYamlVersion, value: v) +proc ts(v: string): TokenWithValue = + TokenWithValue(kind: ltTagShorthand, value: v) +proc tu(v: string): TokenWithValue = + TokenWithValue(kind: ltTagUri, value: v) +proc dirE(): TokenWithValue = TokenWithValue(kind: ltDirectivesEnd) +proc docE(): TokenWithValue = TokenWithValue(kind: ltDocumentEnd) +proc bsh(): TokenWithValue = TokenWithValue(kind: ltBlockScalarHeader) +proc bs(v: string): TokenWithValue = + TokenWithValue(kind: ltBlockScalar, value: v) +proc el(): TokenWithValue = TokenWithValue(kind: ltEmptyLine) +proc ao(): TokenWithValue = TokenWithValue(kind: ltBracketOpen) +proc ac(): TokenWithValue = TokenWithValue(kind: ltBracketClose) +proc oo(): TokenWithValue = TokenWithValue(kind: ltBraceOpen) +proc oc(): TokenWithValue = TokenWithValue(kind: ltBraceClose) +proc c(): TokenWithValue = TokenWithValue(kind: ltComma) +proc th(handle, suffix: string): TokenWithValue = + TokenWithValue(kind: ltTagHandle, handle: handle, suffix: suffix) +proc lt(v: string): TokenWithValue = + TokenWithValue(kind: ltLiteralTag, value: v) +proc an(v: string): TokenWithValue = TokenWithValue(kind: ltAnchor, value: v) +proc al(v: string): TokenWithValue = TokenWithValue(kind: ltAlias, value: v) + +suite "Lexer": + test "Empty document": + assertEquals("", se()) + + test "Single-line scalar": + assertEquals("scalar", i(0), sp("scalar"), se()) + + test "Multiline scalar": + assertEquals("scalar\l line two", i(0), sp("scalar"), i(2), + sp("line two"), se()) + + test "Single-line mapping": + assertEquals("key: value", i(0), sp("key"), mv(), sp("value"), se()) + + test "Multiline mapping": + assertEquals("key:\n value", i(0), sp("key"), mv(), i(2), sp("value"), + se()) + + test "Explicit mapping": + assertEquals("? key\n: value", i(0), mk(), sp("key"), i(0), mv(), + sp("value"), se()) + + test "Sequence": + assertEquals("- a\n- b", i(0), si(), sp("a"), i(0), si(), sp("b"), se()) + + test "Single-line single-quoted scalar": + assertEquals("'quoted scalar'", i(0), qs("quoted scalar"), se()) + + test "Multiline single-quoted scalar": + assertEquals("'quoted\l multi line \l\lscalar'", i(0), + qs("quoted multi line\lscalar"), se()) + + test "Single-line double-quoted scalar": + assertEquals("\"quoted scalar\"", i(0), qs("quoted scalar"), se()) + + test "Multiline double-quoted scalar": + assertEquals("\"quoted\l multi line \l\lscalar\"", i(0), + qs("quoted multi line\lscalar"), se()) + + test "Escape sequences": + assertEquals(""""\n\x31\u0032\U00000033"""", i(0), qs("\l123"), se()) + + test "Directives": + assertEquals("%YAML 1.2\n---\n%TAG\n...\n\n%TAG ! example.html", + dy(), yv("1.2"), dirE(), i(0), sp("%TAG"), i(0), docE(), dt(), + ts("!"), tu("example.html"), se()) + + test "Markers and Unknown Directive": + assertEquals("---\n---\n...\n%UNKNOWN warbl", dirE(), dirE(), i(0), + docE(), du("UNKNOWN"), dp("warbl"), se()) + + test "Block scalar": + assertEquals("|\l a\l\l b\l # comment", i(0), bsh(), bs("a\l\lb\l"), se()) + + test "Block Scalars": + assertEquals("one : >2-\l foo\l bar\ltwo: |+\l bar\l baz", i(0), + sp("one"), mv(), bsh(), bs(" foo\lbar"), i(0), sp("two"), mv(), bsh(), + bs("bar\l baz"), se()) + + test "Flow indicators": + assertEquals("bla]: {c: d, [e]: f}", i(0), sp("bla]"), mv(), oo(), sp("c"), + mv(), sp("d"), c(), ao(), sp("e"), ac(), mv(), sp("f"), oc(), se()) + + test "Adjacent map values in flow style": + assertEquals("{\"foo\":bar, [1]\l:egg}", i(0), oo(), qs("foo"), mv(), + sp("bar"), c(), ao(), sp("1"), ac(), mv(), sp("egg"), oc(), se()) + + test "Tag handles": + assertEquals("- !!str string\l- !local local\l- !e! e", i(0), si(), + th("!!", "str"), sp("string"), i(0), si(), th("!", "local"), + sp("local"), i(0), si(), th("!e!", ""), sp("e"), se()) + + test "Literal tag handle": + assertEquals("! string", i(0), + lt("tag:yaml.org,2002:str"), sp("string"), se()) + + test "Anchors and aliases": + assertEquals("&a foo: {&b b: *a, *b : c}", i(0), an("a"), sp("foo"), mv(), + oo(), an("b"), sp("b"), mv(), al("a"), c(), al("b"), mv(), sp("c"), + oc(), se()) + + test "Empty lines": + assertEquals("""block: foo + + bar + + baz +flow: { + foo + + bar: baz + + + mi +}""", i(0), sp("block"), mv(), sp("foo"), el(), i(2), sp("bar"), el(), i(4), + sp("baz"), i(0), sp("flow"), mv(), oo(), sp("foo"), el(), sp("bar"), mv(), + sp("baz"), el(), el(), sp("mi"), oc(), se()) + +suite "Lookahead": + test "Simple Scalar": + assertLookahead("abcde", false) + + test "Simple Mapping": + assertLookahead("a: b", true) + + test "Colon inside plain scalar": + assertLookahead("abc:de", false) + + test "Colon inside quoted scalar": + assertLookahead("'abc: de'", false) + + test "Quotes inside plain scalar": + assertLookahead("abc\'\"de: foo", true) + + test "Flow indicator inside plain scalar": + assertLookahead("abc}]: de", true) + + test "Complex key": + assertLookahead("[1, 2, \"3\"]: foo", true) + + test "Flow value": + assertLookahead("{a: b}", false) + + test "In flow context": + assertLookahead("[ abcde]: foo", false, 2) + + test "Adjacent value": + assertLookahead("[\"abc\":de]", true, 2) \ No newline at end of file diff --git a/test/serializing.nim b/test/tserialization.nim similarity index 57% rename from test/serializing.nim rename to test/tserialization.nim index cdbb040..861dbc0 100644 --- a/test/serializing.nim +++ b/test/tserialization.nim @@ -5,31 +5,31 @@ # distribution, for details about the copyright. import "../yaml" -import unittest, strutils +import unittest, strutils, streams, tables type MyTuple = tuple str: string i: int32 b: bool - + TrafficLight = enum tlGreen, tlYellow, tlRed - + Person = object firstnamechar: char surname: string age: int32 - + Node = object value: string next: ref Node - + BetterInt = distinct int - + AnimalKind = enum akCat, akDog - + Animal = object name: string case kind: AnimalKind @@ -39,22 +39,21 @@ type barkometer: int proc `$`(v: BetterInt): string {.borrow.} -proc `==`(l, r: BetterInt): bool {.borrow.} +proc `==`(left, right: BetterInt): bool {.borrow.} setTagUri(TrafficLight, "!tl") setTagUri(Node, "!example.net:Node") setTagUri(BetterInt, "!test:BetterInt") proc representObject*(value: BetterInt, ts: TagStyle = tsNone, - c: SerializationContext, tag: TagId): RawYamlStream {.raises: [].} = - result = iterator(): YamlStreamEvent = - var - val = $value - i = val.len - 3 - while i > 0: - val.insert("_", i) - i -= 3 - yield scalarEvent(val, tag, yAnchorNone) + c: SerializationContext, tag: TagId) {.raises: [].} = + var + val = $value + i = val.len - 3 + while i > 0: + val.insert("_", i) + i -= 3 + c.put(scalarEvent(val, tag, yAnchorNone)) proc constructObject*(s: var YamlStream, c: ConstructionContext, result: var BetterInt) @@ -70,127 +69,179 @@ template assertStringEqual(expected, actual: string) = echo "expected:\n", expected, "\nactual:\n", actual assert(false) +template expectConstructionError(li, co: int, message: string, body: typed) = + try: + body + echo "Expected YamlConstructionError, but none was raised!" + fail() + except YamlConstructionError: + let e = (ref YamlConstructionError)(getCurrentException()) + doAssert li == e.line, "Expected error line " & $li & ", was " & $e.line + doAssert co == e.column, "Expected error column " & $co & ", was " & $e.column + doAssert message == e.msg, "Expected error message " & escape(message) & + ", got " & escape(e.msg) + proc newNode(v: string): ref Node = new(result) result.value = v result.next = nil -suite "Serialization": - setup: - let blockOnly = defineOptions(style=psBlockOnly) +let blockOnly = defineOptions(style=psBlockOnly) - test "Serialization: Load integer without fixed length": +suite "Serialization": + test "Load integer without fixed length": var input = newStringStream("-4247") var result: int load(input, result) - assert result == -4247, "result is " & $result - + assert result == -4247, "result is " & $result + input = newStringStream($(int64(int32.high) + 1'i64)) var gotException = false try: load(input, result) except: gotException = true assert gotException, "Expected exception, got none." - test "Serialization: Dump integer without fixed length": + test "Dump integer without fixed length": var input = -4247 - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n\"-4247\"", output.data - + var output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual "%YAML 1.2\n--- \n\"-4247\"", output + when sizeof(int) == sizeof(int64): input = int(int32.high) + 1 var gotException = false - try: dump(input, output, tsNone, asTidy, blockOnly) + try: output = dump(input, tsNone, asTidy, blockOnly) except: gotException = true assert gotException, "Expected exception, got none." - test "Serialization: Load nil string": + test "Load Hex byte (0xFF)": + let input = newStringStream("0xFF") + var result: byte + load(input, result) + assert(result == 255) + + test "Load Hex byte (0xC)": + let input = newStringStream("0xC") + var result: byte + load(input, result) + assert(result == 12) + + test "Load Octal byte (0o14)": + let input = newStringStream("0o14") + var result: byte + load(input, result) + assert(result == 12) + + test "Load byte (14)": + let input = newStringStream("14") + var result: byte + load(input, result) + assert(result == 14) + + test "Load Hex int (0xFF)": + let input = newStringStream("0xFF") + var result: int + load(input, result) + assert(result == 255) + + test "Load Hex int (0xC)": + let input = newStringStream("0xC") + var result: int + load(input, result) + assert(result == 12) + + test "Load Octal int (0o14)": + let input = newStringStream("0o14") + var result: int + load(input, result) + assert(result == 12) + + test "Load int (14)": + let input = newStringStream("14") + var result: int + load(input, result) + assert(result == 14) + + test "Load nil string": let input = newStringStream("!nim:nil:string \"\"") var result: string load(input, result) assert isNil(result) - - test "Serialization: Dump nil string": - let input: string = nil - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n!nim:nil:string \"\"", output.data - test "Serialization: Load string sequence": + test "Dump nil string": + let input: string = nil + var output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual "%YAML 1.2\n--- \n!nim:nil:string \"\"", output + + test "Load string sequence": let input = newStringStream(" - a\n - b") var result: seq[string] load(input, result) assert result.len == 2 assert result[0] == "a" assert result[1] == "b" - - test "Serialization: Dump string sequence": + + test "Dump string sequence": var input = @["a", "b"] - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output.data - - test "Serialization: Load nil seq": + var output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output + + test "Load nil seq": let input = newStringStream("!nim:nil:seq \"\"") var result: seq[int] load(input, result) assert isNil(result) - - test "Serialization: Dump nil seq": + + test "Dump nil seq": let input: seq[int] = nil - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n!nim:nil:seq \"\"", output.data - - test "Serialization: Load char set": + var output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual "%YAML 1.2\n--- \n!nim:nil:seq \"\"", output + + test "Load char set": let input = newStringStream("- a\n- b") var result: set[char] load(input, result) assert result.card == 2 assert 'a' in result assert 'b' in result - - test "Serialization: Dump char set": + + test "Dump char set": var input = {'a', 'b'} - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output.data - - test "Serialization: Load array": + var output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output + + test "Load array": let input = newStringStream("- 23\n- 42\n- 47") var result: array[0..2, int32] load(input, result) assert result[0] == 23 assert result[1] == 42 assert result[2] == 47 - - test "Serialization: Dump array": + + test "Dump array": let input = [23'i32, 42'i32, 47'i32] - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n- 23\n- 42\n- 47", output.data - - test "Serialization: Load Table[int, string]": + var output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual "%YAML 1.2\n--- \n- 23\n- 42\n- 47", output + + test "Load Table[int, string]": let input = newStringStream("23: dreiundzwanzig\n42: zweiundvierzig") var result: Table[int32, string] load(input, result) assert result.len == 2 assert result[23] == "dreiundzwanzig" assert result[42] == "zweiundvierzig" - - test "Serialization: Dump Table[int, string]": + + test "Dump Table[int, string]": var input = initTable[int32, string]() input[23] = "dreiundzwanzig" input[42] = "zweiundvierzig" - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) + var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual("%YAML 1.2\n--- \n23: dreiundzwanzig\n42: zweiundvierzig", - output.data) - - test "Serialization: Load OrderedTable[tuple[int32, int32], string]": + output) + + test "Load OrderedTable[tuple[int32, int32], string]": let input = newStringStream("- {a: 23, b: 42}: drzw\n- {a: 13, b: 47}: drsi") var result: OrderedTable[tuple[a, b: int32], string] - load(input, result) + load(input, result) var i = 0 for key, value in result.pairs: case i @@ -202,13 +253,12 @@ suite "Serialization": assert value == "drsi" else: assert false i.inc() - - test "Serialization: Dump OrderedTable[tuple[int32, int32], string]": + + test "Dump OrderedTable[tuple[int32, int32], string]": var input = initOrderedTable[tuple[a, b: int32], string]() input.add((a: 23'i32, b: 42'i32), "dreiundzwanzigzweiundvierzig") input.add((a: 13'i32, b: 47'i32), "dreizehnsiebenundvierzig") - var output = newStringStream() - dump(input, output, tsRootOnly, asTidy, blockOnly) + var output = dump(input, tsRootOnly, asTidy, blockOnly) assertStringEqual("""%YAML 1.2 --- !nim:tables:OrderedTable(nim:tuple(nim:system:int32,nim:system:int32),tag:yaml.org,2002:str) - @@ -220,9 +270,9 @@ suite "Serialization": ? a: 13 b: 47 - : dreizehnsiebenundvierzig""", output.data) - - test "Serialization: Load Sequences in Sequence": + : dreizehnsiebenundvierzig""", output) + + test "Load Sequences in Sequence": let input = newStringStream(" - [1, 2, 3]\n - [4, 5]\n - [6]") var result: seq[seq[int32]] load(input, result) @@ -230,31 +280,28 @@ suite "Serialization": assert result[0] == @[1.int32, 2.int32, 3.int32] assert result[1] == @[4.int32, 5.int32] assert result[2] == @[6.int32] - - test "Serialization: Dump Sequences in Sequence": + + test "Dump Sequences in Sequence": let input = @[@[1.int32, 2.int32, 3.int32], @[4.int32, 5.int32], @[6.int32]] - var output = newStringStream() - dump(input, output, tsNone) - assertStringEqual "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]", - output.data - - test "Serialization: Load Enum": - let input = newStringStream("!nim:system:seq(tl)\n- !tl tlRed\n- tlGreen\n- tlYellow") + var output = dump(input, tsNone) + assertStringEqual "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]", output + + test "Load Enum": + let input = + newStringStream("!nim:system:seq(tl)\n- !tl tlRed\n- tlGreen\n- tlYellow") var result: seq[TrafficLight] load(input, result) assert result.len == 3 assert result[0] == tlRed assert result[1] == tlGreen assert result[2] == tlYellow - - test "Serialization: Dump Enum": + + test "Dump Enum": let input = @[tlRed, tlGreen, tlYellow] - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) - assertStringEqual "%YAML 1.2\n--- \n- tlRed\n- tlGreen\n- tlYellow", - output.data - - test "Serialization: Load Tuple": + var output = dump(input, tsNone, asTidy, blockOnly) + assertStringEqual "%YAML 1.2\n--- \n- tlRed\n- tlGreen\n- tlYellow", output + + test "Load Tuple": let input = newStringStream("str: value\ni: 42\nb: true") var result: MyTuple load(input, result) @@ -262,43 +309,91 @@ suite "Serialization": assert result.i == 42 assert result.b == true - test "Serialization: Dump Tuple": + test "Dump Tuple": let input = (str: "value", i: 42.int32, b: true) - var output = newStringStream() - dump(input, output, tsNone) - assertStringEqual "%YAML 1.2\n--- \nstr: value\ni: 42\nb: y", output.data - - test "Serialization: Load custom object": + var output = dump(input, tsNone) + assertStringEqual "%YAML 1.2\n--- \nstr: value\ni: 42\nb: y", output + + test "Load Tuple - unknown field": + let input = "str: value\nfoo: bar\ni: 42\nb: true" + var result: MyTuple + expectConstructionError(2, 1, "While constructing MyTuple: Unknown field: \"foo\""): + load(input, result) + + test "Load Tuple - missing field": + let input = "str: value\nb: true" + var result: MyTuple + expectConstructionError(2, 8, "While constructing MyTuple: Missing field: \"i\""): + load(input, result) + + test "Load Tuple - duplicate field": + let input = "str: value\ni: 42\nb: true\nb: true" + var result: MyTuple + expectConstructionError(4, 1, "While constructing MyTuple: Duplicate field: \"b\""): + load(input, result) + + test "Load Multiple Documents": + let input = newStringStream("1\n---\n2") + var result: seq[int] + loadMultiDoc(input, result) + assert(result.len == 2) + assert result[0] == 1 + assert result[1] == 2 + + test "Load Multiple Documents (Single Doc)": + let input = newStringStream("1") + var result: seq[int] + loadMultiDoc(input, result) + assert(result.len == 1) + assert result[0] == 1 + + test "Load custom object": let input = newStringStream("firstnamechar: P\nsurname: Pan\nage: 12") var result: Person load(input, result) assert result.firstnamechar == 'P' assert result.surname == "Pan" assert result.age == 12 - - test "Serialization: Dump custom object": + + test "Dump custom object": let input = Person(firstnamechar: 'P', surname: "Pan", age: 12) - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) + var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual( - "%YAML 1.2\n--- \nfirstnamechar: P\nsurname: Pan\nage: 12", output.data) - - test "Serialization: Load sequence with explicit tags": + "%YAML 1.2\n--- \nfirstnamechar: P\nsurname: Pan\nage: 12", output) + + test "Load custom object - unknown field": + let input = " firstnamechar: P\n surname: Pan\n age: 12\n occupation: free" + var result: Person + expectConstructionError(4, 3, "While constructing Person: Unknown field: \"occupation\""): + load(input, result) + + test "Load custom object - missing field": + let input = "surname: Pan\nage: 12\n " + var result: Person + expectConstructionError(3, 3, "While constructing Person: Missing field: \"firstnamechar\""): + load(input, result) + + test "Load custom object - duplicate field": + let input = "firstnamechar: P\nsurname: Pan\nage: 12\nsurname: Pan" + var result: Person + expectConstructionError(4, 1, "While constructing Person: Duplicate field: \"surname\""): + load(input, result) + + test "Load sequence with explicit tags": let input = newStringStream("--- !nim:system:seq(" & "tag:yaml.org,2002:str)\n- !!str one\n- !!str two") var result: seq[string] load(input, result) assert result[0] == "one" assert result[1] == "two" - - test "Serialization: Dump sequence with explicit tags": + + test "Dump sequence with explicit tags": let input = @["one", "two"] - var output = newStringStream() - dump(input, output, tsAll, asTidy, blockOnly) + var output = dump(input, tsAll, asTidy, blockOnly) assertStringEqual("%YAML 1.2\n--- !nim:system:seq(" & - "tag:yaml.org,2002:str) \n- !!str one\n- !!str two", output.data) - - test "Serialization: Load custom object with explicit root tag": + "tag:yaml.org,2002:str) \n- !!str one\n- !!str two", output) + + test "Load custom object with explicit root tag": let input = newStringStream( "--- !nim:custom:Person\nfirstnamechar: P\nsurname: Pan\nage: 12") var result: Person @@ -306,16 +401,15 @@ suite "Serialization": assert result.firstnamechar == 'P' assert result.surname == "Pan" assert result.age == 12 - - test "Serialization: Dump custom object with explicit root tag": + + test "Dump custom object with explicit root tag": let input = Person(firstnamechar: 'P', surname: "Pan", age: 12) - var output = newStringStream() - dump(input, output, tsRootOnly, asTidy, blockOnly) + var output = dump(input, tsRootOnly, asTidy, blockOnly) assertStringEqual("%YAML 1.2\n" & "--- !nim:custom:Person \nfirstnamechar: P\nsurname: Pan\nage: 12", - output.data) - - test "Serialization: Load custom variant object": + output) + + test "Load custom variant object": let input = newStringStream( "---\n- - name: Bastet\n - kind: akCat\n - purringIntensity: 7\n" & "- - name: Anubis\n - kind: akDog\n - barkometer: 13") @@ -328,12 +422,11 @@ suite "Serialization": assert result[1].name == "Anubis" assert result[1].kind == akDog assert result[1].barkometer == 13 - - test "Serialization: Dump custom variant object": + + test "Dump custom variant object": let input = @[Animal(name: "Bastet", kind: akCat, purringIntensity: 7), Animal(name: "Anubis", kind: akDog, barkometer: 13)] - var output = newStringStream() - dump(input, output, tsNone, asTidy, blockOnly) + var output = dump(input, tsNone, asTidy, blockOnly) assertStringEqual """%YAML 1.2 --- - @@ -349,9 +442,15 @@ suite "Serialization": - kind: akDog - - barkometer: 13""", output.data - - test "Serialization: Dump cyclic data structure": + barkometer: 13""", output + + test "Load custom variant object - missing field": + let input = "[{name: Bastet}, {kind: akCat}]" + var result: Animal + expectConstructionError(1, 32, "While constructing Animal: Missing field: \"purringIntensity\""): + load(input, result) + + test "Dump cyclic data structure": var a = newNode("a") b = newNode("b") @@ -359,8 +458,7 @@ suite "Serialization": a.next = b b.next = c c.next = a - var output = newStringStream() - dump(a, output, tsRootOnly, asTidy, blockOnly) + var output = dump(a, tsRootOnly, asTidy, blockOnly) assertStringEqual """%YAML 1.2 --- !example.net:Node &a value: a @@ -368,16 +466,16 @@ next: value: b next: value: c - next: *a""", output.data - - test "Serialization: Load cyclic data structure": + next: *a""", output + + test "Load cyclic data structure": let input = newStringStream("""%YAML 1.2 ---- !nim:system:seq(example.net:Node) -- &a +--- !nim:system:seq(example.net:Node) +- &a value: a - next: &b + next: &b value: b - next: &c + next: &c value: c next: *a - *b @@ -398,8 +496,8 @@ next: assert(result[0].next == result[1]) assert(result[1].next == result[2]) assert(result[2].next == result[0]) - - test "Serialization: Load nil values": + + test "Load nil values": let input = newStringStream("- ~\n- !!str ~") var result: seq[ref string] try: load(input, result) @@ -408,36 +506,34 @@ next: echo "line ", ex.line, ", column ", ex.column, ": ", ex.msg echo ex.lineContent raise ex - + assert(result.len == 2) assert(result[0] == nil) assert(result[1][] == "~") - - test "Serialization: Dump nil values": + + test "Dump nil values": var input = newSeq[ref string]() input.add(nil) input.add(new string) input[1][] = "~" - var output = newStringStream() - dump(input, output, tsRootOnly, asTidy, blockOnly) + var output = dump(input, tsRootOnly, asTidy, blockOnly) assertStringEqual( "%YAML 1.2\n--- !nim:system:seq(tag:yaml.org,2002:str) \n- !!null ~\n- !!str ~", - output.data) - - test "Serialization: Custom constructObject": + output) + + test "Custom constructObject": let input = newStringStream("- 1\n- !test:BetterInt 2") var result: seq[BetterInt] load(input, result) assert(result.len == 2) assert(result[0] == 2.BetterInt) assert(result[1] == 3.BetterInt) - - test "Serialization: Custom representObject": + + test "Custom representObject": let input = @[1.BetterInt, 9998887.BetterInt, 98312.BetterInt] - var output = newStringStream() - dump(input, output, tsAll, asTidy, blockOnly) + var output = dump(input, tsAll, asTidy, blockOnly) assertStringEqual """%YAML 1.2 --- !nim:system:seq(test:BetterInt) - !test:BetterInt 1 - !test:BetterInt 9_998_887 -- !test:BetterInt 98_312""", output.data \ No newline at end of file +- !test:BetterInt 98_312""", output diff --git a/test/yamlTestSuite.nim b/test/yamlTestSuite.nim index 4824618..2423af4 100644 --- a/test/yamlTestSuite.nim +++ b/test/yamlTestSuite.nim @@ -4,8 +4,8 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. -import os, terminal, strutils -import testEventParser, common +import os, terminal, strutils, streams +import testEventParser, commonTestUtils import "../yaml" const gitCmd = diff --git a/yaml.nim b/yaml.nim index 94f3bd3..0703b65 100644 --- a/yaml.nim +++ b/yaml.nim @@ -1,761 +1,44 @@ # NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause +# (c) Copyright 2016 Felix Krause # # See the file "copying.txt", included in this # distribution, for details about the copyright. -## This module provides facilities to generate and interpret -## `YAML `_ character streams. All primitive operations on -## data objects use a `YamlStream <#YamlStream>`_ either as source or as -## output. Because this stream is implemented as iterator, it is possible to -## process YAML input and output sequentially, i.e. without loading the -## processed data structure completely into RAM. This supports the processing of -## large data structures. +## This is the parent module of NimYAML, a package that provides facilities to +## generate and interpret `YAML `_ character streams. Importing +## this package will import everything from all subpackages. ## -## As YAML is a strict superset of `JSON `_, JSON input is -## automatically supported. While JSON is less readable than YAML, -## this enhances interoperability with other languages. +## There are three high-level APIs which are probably most useful: +## +## * The serialization API in `serialization `_ enables +## you to load YAML data directly into native Nim types, and reversely dump +## native Nim types as YAML. +## * The DOM API in `dom `_ parses YAML files in a tree structure +## which you can navigate. +## * The JSON API in `tojson `_ parses YAML files into the +## Nim stdlib's JSON structure, which may be useful if you have other modules +## which expect JSON input. Note that the serialization API is able to write +## and load JSON; you do not need the JSON API for that. +## +## Apart from those high-level APIs, NimYAML implements a low-level API which +## enables you to process YAML input as data stream which does not need to be +## loaded into RAM completely at once. It consists of the following modules: +## +## * The stream API in `stream `_ defines the central type for +## stream processing, ``YamlStream``. It also contains definitions and +## constructor procs for stream events. +## * The parser API in `parser `_ gives you direct access to +## the YAML parser's output. +## * The presenter API in `presenter `_ gives you direct +## access to the presenter, i.e. the module that renders a YAML character +## stream. +## * The taglib API in `taglib `_ provides a data structure +## for keeping track of YAML tags that are generated by the parser or used in +## the presenter. +## * The hints API in `hints `_ provides a simple proc for +## guessing the type of a scalar value. -import streams, unicode, lexbase, tables, strutils, json, hashes, queues, - macros, typetraits, parseutils -export streams, tables, json - -when defined(yamlDebug): import terminal - -type - TypeHint* = enum - ## A type hint can be computed from scalar content and tells you what - ## NimYAML thinks the scalar's type is. It is generated by - ## `guessType <#guessType,string>`_ The first matching RegEx - ## in the following table will be the type hint of a scalar string. - ## - ## You can use it to determine the type of YAML scalars that have a '?' - ## non-specific tag, but using this feature is completely optional. - ## - ## ================== ========================= - ## Name RegEx - ## ================== ========================= - ## ``yTypeInteger`` ``0 | -? [1-9] [0-9]*`` - ## ``yTypeFloat`` ``-? [1-9] ( \. [0-9]* [1-9] )? ( e [-+] [1-9] [0-9]* )?`` - ## ``yTypeFloatInf`` ``-? \. (inf | Inf | INF)`` - ## ``yTypeFloatNaN`` ``-? \. (nan | NaN | NAN)`` - ## ``yTypeBoolTrue`` ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON`` - ## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF`` - ## ``yTypeNull`` ``~ | null | Null | NULL`` - ## ``yTypeUnknown`` ``*`` - ## ================== ========================= - yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue, - yTypeBoolFalse, yTypeNull, yTypeUnknown - - YamlStreamEventKind* = enum - ## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds - ## are discussed in `YamlStreamEvent <#YamlStreamEvent>`_. - yamlStartDoc, yamlEndDoc, yamlStartMap, yamlEndMap, - yamlStartSeq, yamlEndSeq, yamlScalar, yamlAlias - - TagId* = distinct int ## \ - ## A ``TagId`` identifies a tag URI, like for example - ## ``"tag:yaml.org,2002:str"``. The URI corresponding to a ``TagId`` can - ## be queried from the `TagLibrary <#TagLibrary>`_ which was - ## used to create this ``TagId``; e.g. when you parse a YAML character - ## stream, the ``TagLibrary`` of the parser is the one which generates - ## the resulting ``TagId`` s. - ## - ## URI strings are mapped to ``TagId`` s for efficiency reasons (you - ## do not need to compare strings every time) and to be able to - ## discover unknown tag URIs early in the parsing process. - - AnchorId* = distinct int ## \ - ## An ``AnchorId`` identifies an anchor in the current document. It - ## becomes invalid as soon as the current document scope is invalidated - ## (for example, because the parser yielded a ``yamlEndDocument`` - ## event). ``AnchorId`` s exists because of efficiency, much like - ## ``TagId`` s. The actual anchor name is a presentation detail and - ## cannot be queried by the user. - - YamlStreamEvent* = object - ## An element from a `YamlStream <#YamlStream>`_. Events that start an - ## object (``yamlStartMap``, ``yamlStartSeq``, ``yamlScalar``) have - ## an optional anchor and a tag associated with them. The anchor will be - ## set to ``yAnchorNone`` if it doesn't exist. - ## - ## A non-existing tag in the YAML character stream will be resolved to - ## the non-specific tags ``?`` or ``!`` according to the YAML - ## specification. These are by convention mapped to the ``TagId`` s - ## ``yTagQuestionMark`` and ``yTagExclamationMark`` respectively. - ## Mapping is done by a `TagLibrary <#TagLibrary>`_. - case kind*: YamlStreamEventKind - of yamlStartMap: - mapAnchor* : AnchorId - mapTag* : TagId - of yamlStartSeq: - seqAnchor* : AnchorId - seqTag* : TagId - of yamlScalar: - scalarAnchor* : AnchorId - scalarTag* : TagId - scalarContent*: string # may not be nil (but empty) - of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard - of yamlAlias: - aliasTarget* : AnchorId - - YamlStream* = object ## \ - ## A ``YamlStream`` is an iterator-like object that yields a - ## well-formed stream of ``YamlStreamEvents``. Well-formed means that - ## every ``yamlStartMap`` is terminated by a ``yamlEndMap``, every - ## ``yamlStartSeq`` is terminated by a ``yamlEndSeq`` and every - ## ``yamlStartDoc`` is terminated by a ``yamlEndDoc``. Moreover, every - ## emitted mapping has an even number of children. - ## - ## The creator of a ``YamlStream`` is responsible for it being - ## well-formed. A user of the stream may assume that it is well-formed - ## and is not required to check for it. The procs in this module will - ## always yield a well-formed ``YamlStream`` and expect it to be - ## well-formed if they take it as input parameter. - ## - ## - backend: iterator(): YamlStreamEvent - peeked: bool - cached: YamlStreamEvent - - TagLibrary* = ref object - ## A ``TagLibrary`` maps tag URIs to ``TagId`` s. - ## - ## When `YamlParser <#YamlParser>`_ encounters tags not existing in the - ## tag library, it will use - ## `registerUri <#registerUri,TagLibrary,string>`_ to add - ## the tag to the library. - ## - ## You can base your tag library on common tag libraries by initializing - ## them with `initFailsafeTagLibrary <#initFailsafeTagLibrary>`_, - ## `initCoreTagLibrary <#initCoreTagLibrary>`_ or - ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_. - tags*: Table[string, TagId] - nextCustomTagId*: TagId - secondaryPrefix*: string - - - WarningCallback* = proc(line, column: int, lineContent: string, - message: string) - ## Callback for parser warnings. Currently, this callback may be called - ## on two occasions while parsing a YAML document stream: - ## - ## - If the version number in the ``%YAML`` directive does not match - ## ``1.2``. - ## - If there is an unknown directive encountered. - - FastParseLevelKind = enum - fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey, - fplSinglePairValue, fplScalar, fplDocument - - FastParseLevel = object - kind: FastParseLevelKind - indentation: int - - YamlParser* = ref object - ## A parser object. Retains its ``TagLibrary`` across calls to - ## `parse <#parse,YamlParser,Stream>`_. Can be used - ## to access anchor names while parsing a YAML character stream, but - ## only until the document goes out of scope (i.e. until - ## ``yamlEndDocument`` is yielded). - tagLib: TagLibrary - callback: WarningCallback - lexer: BaseLexer - tokenstart: int - content, after: string - ancestry: seq[FastParseLevel] - level: FastParseLevel - tagUri: string - tag: TagId - anchor: AnchorId - shorthands: Table[string, string] - anchors: Table[string, AnchorId] - nextAnchorId: AnchorId - newlines: int - indentation: int - - PresentationStyle* = enum - ## Different styles for YAML character stream output. - ## - ## - ``ypsMinimal``: Single-line flow-only output which tries to - ## use as few characters as possible. - ## - ``ypsCanonical``: Canonical YAML output. Writes all tags except - ## for the non-specific tags ``?`` and ``!``, uses flow style, quotes - ## all string scalars. - ## - ``ypsDefault``: Tries to be as human-readable as possible. Uses - ## block style by default, but tries to condense mappings and - ## sequences which only contain scalar nodes into a single line using - ## flow style. - ## - ``ypsJson``: Omits the ``%YAML`` directive and the ``---`` - ## marker. Uses flow style. Flattens anchors and aliases, omits tags. - ## Output will be parseable as JSON. ``YamlStream`` to dump may only - ## contain one document. - ## - ``ypsBlockOnly``: Formats all output in block style, does not use - ## flow style at all. - psMinimal, psCanonical, psDefault, psJson, psBlockOnly - - TagStyle* = enum - ## Whether object should be serialized with explicit tags. - ## - ## - ``tsNone``: No tags will be outputted unless necessary. - ## - ``tsRootOnly``: A tag will only be outputted for the root tag and - ## where necessary. - ## - ``tsAll``: Tags will be outputted for every object. - tsNone, tsRootOnly, tsAll - - AnchorStyle* = enum - ## How ref object should be serialized. - ## - ## - ``asNone``: No anchors will be outputted. Values present at - ## multiple places in the content that should be serialized will be - ## fully serialized at every occurence. If the content is cyclic, this - ## will lead to an endless loop! - ## - ``asTidy``: Anchors will only be generated for objects that - ## actually occur more than once in the content to be serialized. - ## This is a bit slower and needs more memory than ``asAlways``. - ## - ``asAlways``: Achors will be generated for every ref object in the - ## content to be serialized, regardless of whether the object is - ## referenced again afterwards - asNone, asTidy, asAlways - - NewLineStyle* = enum - ## What kind of newline sequence is used when presenting. - ## - ## - ``nlLF``: Use a single linefeed char as newline. - ## - ``nlCRLF``: Use a sequence of carriage return and linefeed as - ## newline. - ## - ``nlOSDefault``: Use the target operation system's default newline - ## sequence (CRLF on Windows, LF everywhere else). - nlLF, nlCRLF, nlOSDefault - - OutputYamlVersion* = enum - ## Specify which YAML version number the presenter shall emit. The - ## presenter will always emit content that is valid YAML 1.1, but by - ## default will write a directive ``%YAML 1.2``. For compatibility with - ## other YAML implementations, it is possible to change this here. - ## - ## It is also possible to specify that the presenter shall not emit any - ## YAML version. The generated content is then guaranteed to be valid - ## YAML 1.1 and 1.2 (but not 1.0 or any newer YAML version). - ov1_2, ov1_1, ovNone - - PresentationOptions* = object - ## Options for generating a YAML character stream - style*: PresentationStyle - indentationStep*: int - newlines*: NewLineStyle - outputVersion*: OutputYamlVersion - - ConstructionContext* = ref object - ## Context information for the process of constructing Nim values from YAML. - refs: Table[AnchorId, pointer] - - SerializationContext* = ref object - ## Context information for the process of serializing YAML from Nim values. - refs: Table[pointer, AnchorId] - style: AnchorStyle - nextAnchorId: AnchorId - - RawYamlStream* = iterator(): YamlStreamEvent {.raises: [].} ## \ - ## Stream of ``YamlStreamEvent``s returned by ``representObject`` procs. - - YamlNodeKind* = enum - yScalar, yMapping, ySequence - - YamlNode* = ref YamlNodeObj not nil - ## Represents a node in a ``YamlDocument``. - - YamlNodeObj* = object - tag*: string - case kind*: YamlNodeKind - of yScalar: content*: string - of ySequence: children*: seq[YamlNode] - of yMapping: pairs*: seq[tuple[key, value: YamlNode]] - - YamlDocument* = object - ## Represents a YAML document. - root*: YamlNode - - YamlLoadingError* = object of Exception - ## Base class for all exceptions that may be raised during the process - ## of loading a YAML character stream. - line*: int ## line number (1-based) where the error was encountered - column*: int ## column number (1-based) where the error was encountered - lineContent*: string ## \ - ## content of the line where the error was encountered. Includes a - ## second line with a marker ``^`` at the position where the error - ## was encountered. - - YamlParserError* = object of YamlLoadingError - ## A parser error is raised if the character stream that is parsed is - ## not a valid YAML character stream. This stream cannot and will not be - ## parsed wholly nor partially and all events that have been emitted by - ## the YamlStream the parser provides should be discarded. - ## - ## A character stream is invalid YAML if and only if at least one of the - ## following conditions apply: - ## - ## - There are invalid characters in an element whose contents is - ## restricted to a limited set of characters. For example, there are - ## characters in a tag URI which are not valid URI characters. - ## - An element has invalid indentation. This can happen for example if - ## a block list element indicated by ``"- "`` is less indented than - ## the element in the previous line, but there is no block sequence - ## list open at the same indentation level. - ## - The YAML structure is invalid. For example, an explicit block map - ## indicated by ``"? "`` and ``": "`` may not suddenly have a block - ## sequence item (``"- "``) at the same indentation level. Another - ## possible violation is closing a flow style object with the wrong - ## closing character (``}``, ``]``) or not closing it at all. - ## - A custom tag shorthand is used that has not previously been - ## declared with a ``%TAG`` directive. - ## - Multiple tags or anchors are defined for the same node. - ## - An alias is used which does not map to any anchor that has - ## previously been declared in the same document. - ## - An alias has a tag or anchor associated with it. - ## - ## Some elements in this list are vague. For a detailed description of a - ## valid YAML character stream, see the YAML specification. - - YamlPresenterJsonError* = object of Exception - ## Exception that may be raised by the YAML presenter when it is - ## instructed to output JSON, but is unable to do so. This may occur if: - ## - ## - The given `YamlStream <#YamlStream>`_ contains a map which has any - ## non-scalar type as key. - ## - Any float scalar bears a ``NaN`` or positive/negative infinity value - - YamlPresenterOutputError* = object of Exception - ## Exception that may be raised by the YAML presenter. This occurs if - ## writing character data to the output stream raises any exception. - ## The error that has occurred is available from ``parent``. - - YamlStreamError* = object of Exception - ## Exception that may be raised by a ``YamlStream`` when the underlying - ## backend raises an exception. The error that has occurred is - ## available from ``parent``. - - YamlConstructionError* = object of YamlLoadingError - ## Exception that may be raised when constructing data objects from a - ## `YamlStream <#YamlStream>`_. The fields ``line``, ``column`` and - ## ``lineContent`` are only available if the costructing proc also does - ## parsing, because otherwise this information is not available to the - ## costruction proc. - -const - # failsafe schema - - yTagExclamationMark*: TagId = 0.TagId ## ``!`` non-specific tag - yTagQuestionMark* : TagId = 1.TagId ## ``?`` non-specific tag - yTagString* : TagId = 2.TagId ## \ - ## `!!str `_ tag - yTagSequence* : TagId = 3.TagId ## \ - ## `!!seq `_ tag - yTagMapping* : TagId = 4.TagId ## \ - ## `!!map `_ tag - - # json & core schema - - yTagNull* : TagId = 5.TagId ## \ - ## `!!null `_ tag - yTagBoolean* : TagId = 6.TagId ## \ - ## `!!bool `_ tag - yTagInteger* : TagId = 7.TagId ## \ - ## `!!int `_ tag - yTagFloat* : TagId = 8.TagId ## \ - ## `!!float `_ tag - - # other language-independent YAML types (from http://yaml.org/type/ ) - - yTagOrderedMap* : TagId = 9.TagId ## \ - ## `!!omap `_ tag - yTagPairs* : TagId = 10.TagId ## \ - ## `!!pairs `_ tag - yTagSet* : TagId = 11.TagId ## \ - ## `!!set `_ tag - yTagBinary* : TagId = 12.TagId ## \ - ## `!!binary `_ tag - yTagMerge* : TagId = 13.TagId ## \ - ## `!!merge `_ tag - yTagTimestamp* : TagId = 14.TagId ## \ - ## `!!timestamp `_ tag - yTagValue* : TagId = 15.TagId ## \ - ## `!!value `_ tag - yTagYaml* : TagId = 16.TagId ## \ - ## `!!yaml `_ tag - - yTagNimField* : TagId = 100.TagId ## \ - ## This tag is used in serialization for the name of a field of an - ## object. It may contain any string scalar that is a valid Nim symbol. - - yTagNimNilString* : TagId = 101.TagId ## for strings that are nil - yTagNimNilSeq* : TagId = 102.TagId ## \ - ## for seqs that are nil. This tag is used regardless of the seq's generic - ## type parameter. - - yFirstCustomTagId* : TagId = 1000.TagId ## \ - ## The first ``TagId`` which should be assigned to an URI that does not - ## exist in the ``YamlTagLibrary`` which is used for parsing. - - yAnchorNone*: AnchorId = (-1).AnchorId ## \ - ## yielded when no anchor was defined for a YAML node - - yamlTagRepositoryPrefix* = "tag:yaml.org,2002:" - - defaultPresentationOptions* = - PresentationOptions(style: psDefault, indentationStep: 2, - newlines: nlOSDefault) - -# interface - -proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool {.raises: [].} - ## compares all existing fields of the given items - -proc `$`*(event: YamlStreamEvent): string {.raises: [].} - ## outputs a human-readable string describing the given event - -proc tag*(event: YamlStreamEvent): TagId {.raises: [FieldError].} - -proc startDocEvent*(): YamlStreamEvent {.inline, raises: [].} -proc endDocEvent*(): YamlStreamEvent {.inline, raises: [].} -proc startMapEvent*(tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): - YamlStreamEvent {.inline, raises: [].} -proc endMapEvent*(): YamlStreamEvent {.inline, raises: [].} -proc startSeqEvent*(tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): - YamlStreamEvent {.inline, raises: [].} -proc endSeqEvent*(): YamlStreamEvent {.inline, raises: [].} -proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): - YamlStreamEvent {.inline, raises: [].} -proc aliasEvent*(anchor: AnchorId): YamlStreamEvent {.inline, raises: [].} - -proc `==`*(left, right: TagId): bool {.borrow.} -proc `$`*(id: TagId): string -proc hash*(id: TagId): Hash {.borrow.} - -proc `==`*(left, right: AnchorId): bool {.borrow.} -proc `$`*(id: AnchorId): string {.borrow.} -proc hash*(id: AnchorId): Hash {.borrow.} - -proc initYamlStream*(backend: iterator(): YamlStreamEvent): - YamlStream {.raises: [].} - ## Creates a new ``YamlStream`` that uses the given iterator as backend. -proc next*(s: var YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} - ## Get the next item of the stream. Requires ``finished(s) == true``. - ## If the backend yields an exception, that exception will be encapsulated - ## into a ``YamlStreamError``, which will be raised. -proc peek*(s: var YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} - ## Get the next item of the stream without advancing the stream. - ## Requires ``finished(s) == true``. Handles exceptions of the backend like - ## ``next()``. -proc `peek=`*(s: var YamlStream, value: YamlStreamEvent) {.raises: [].} - ## Set the next item of the stream. Will replace a previously peeked item, - ## if one exists. -proc finished*(s: var YamlStream): bool {.raises: [YamlStreamError].} - ## ``true`` if no more items are available in the stream. Handles exceptions - ## of the backend like ``next()``. -iterator items*(s: var YamlStream): YamlStreamEvent - {.raises: [YamlStreamError].} = - ## Iterate over all items of the stream. You may not use ``peek()`` on the - ## stream while iterating. - if s.peeked: - s.peeked = false - yield s.cached - while true: - var event: YamlStreamEvent - try: - event = s.backend() - if finished(s.backend): break - except AssertionError: raise - except YamlStreamError: - let cur = getCurrentException() - var e = newException(YamlStreamError, cur.msg) - e.parent = cur.parent - raise e - except Exception: - var e = newException(YamlStreamError, getCurrentExceptionMsg()) - e.parent = getCurrentException() - raise e - yield event - -proc initTagLibrary*(): TagLibrary {.raises: [].} - ## initializes the ``tags`` table and sets ``nextCustomTagId`` to - ## ``yFirstCustomTagId``. - -proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} - ## registers a custom tag URI with a ``TagLibrary``. The URI will get - ## the ``TagId`` ``nextCustomTagId``, which will be incremented. - -proc uri*(tagLib: TagLibrary, id: TagId): string {.raises: [KeyError].} - ## retrieve the URI a ``TagId`` maps to. - -proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} - ## Contains only: - ## - ``!`` - ## - ``?`` - ## - ``!!str`` - ## - ``!!map`` - ## - ``!!seq`` -proc initCoreTagLibrary*(): TagLibrary {.raises: [].} - ## Contains everything in ``initFailsafeTagLibrary`` plus: - ## - ``!!null`` - ## - ``!!bool`` - ## - ``!!int`` - ## - ``!!float`` -proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} - ## Contains everything from ``initCoreTagLibrary`` plus: - ## - ``!!omap`` - ## - ``!!pairs`` - ## - ``!!set`` - ## - ``!!binary`` - ## - ``!!merge`` - ## - ``!!timestamp`` - ## - ``!!value`` - ## - ``!!yaml`` - -proc initSerializationTagLibrary(): TagLibrary {.raises: [].} - -proc guessType*(scalar: string): TypeHint {.raises: [].} - ## Parse scalar string according to the RegEx table documented at - ## `TypeHint <#TypeHind>`_. - -proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(), - callback: WarningCallback = nil): YamlParser {.raises: [].} - ## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called - ## whenever the parser yields a warning. - -proc getLineNumber*(p: YamlParser): int {.raises: [].} - ## Get the line number (1-based) of the recently yielded parser token. - ## Useful for error reporting at later loading stages. - -proc getColNumber*(p: YamlParser): int {.raises: [].} - ## Get the column number (1-based) of the recently yielded parser token. - ## Useful for error reporting at later parsing stages. - -proc getLineContent*(p: YamlParser, marker: bool = true): string {.raises: [].} - ## Get the content of the input line containing the recently yielded parser - ## token. Useful for error reporting at later parsing stages. The line will - ## be terminated by ``"\n"``. If ``marker`` is ``true``, a second line will - ## be returned containing a ``^`` at the position of the recent parser - ## token. - -proc parse*(p: YamlParser, s: Stream): YamlStream {.raises: [].} - ## Parse the given stream as YAML character stream. - -proc defineOptions*(style: PresentationStyle = psDefault, - indentationStep: int = 2, - newlines: NewLineStyle = nlOSDefault, - outputVersion: OutputYamlVersion = ov1_2): - PresentationOptions {.raises: [].} - ## Define a set of options for presentation. Convenience proc that requires - ## you to only set those values that should not equal the default. - -proc constructJson*(s: var YamlStream): seq[JsonNode] - {.raises: [YamlConstructionError, YamlStreamError].} - ## Construct an in-memory JSON tree from a YAML event stream. The stream may - ## not contain any tags apart from those in ``coreTagLibrary``. Anchors and - ## aliases will be resolved. Maps in the input must not contain - ## non-scalars as keys. Each element of the result represents one document - ## in the YAML stream. - ## - ## **Warning:** The special float values ``[+-]Inf`` and ``NaN`` will be - ## parsed into Nim's JSON structure without error. However, they cannot be - ## rendered to a JSON character stream, because these values are not part - ## of the JSON specification. Nim's JSON implementation currently does not - ## check for these values and will output invalid JSON when rendering one - ## of these values into a JSON character stream. - -proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [].} - ## Uses `YamlParser <#YamlParser>`_ and - ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree - ## from a YAML character stream. - -proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, - options: PresentationOptions = defaultPresentationOptions) - {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, - YamlStreamError].} - ## Convert ``s`` to a YAML character stream and write it to ``target``. - -proc transform*(input: Stream, output: Stream, - options: PresentationOptions = defaultPresentationOptions) - {.raises: [IOError, YamlParserError, YamlPresenterJsonError, - YamlPresenterOutputError].} - ## Parser ``input`` as YAML character stream and then dump it to ``output`` - ## while resolving non-specific tags to the ones in the YAML core tag - ## library. - -proc constructChild*[T](s: var YamlStream, c: ConstructionContext, - result: var T) - {.raises: [YamlConstructionError, YamlStreamError].} - ## Constructs an arbitrary Nim value from a part of a YAML stream. - ## The stream will advance until after the finishing token that was used - ## for constructing the value. The ``ConstructionContext`` is needed for - ## potential child objects which may be refs. - -proc constructChild*(s: var YamlStream, c: ConstructionContext, - result: var string) - {.raises: [YamlConstructionError, YamlStreamError].} - ## Constructs a Nim value that is a string from a part of a YAML stream. - ## This specialization takes care of possible nil strings. - -proc constructChild*[T](s: var YamlStream, c: ConstructionContext, - result: var seq[T]) - {.raises: [YamlConstructionError, YamlStreamError].} - ## Constructs a Nim value that is a string from a part of a YAML stream. - ## This specialization takes care of possible nil seqs. - -proc constructChild*[O](s: var YamlStream, c: ConstructionContext, - result: var ref O) - {.raises: [YamlConstructionError, YamlStreamError].} - ## Constructs an arbitrary Nim value from a part of a YAML stream. - ## The stream will advance until after the finishing token that was used - ## for constructing the value. The object may be constructed from an alias - ## node which will be resolved using the ``ConstructionContext``. - -proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext): - RawYamlStream {.raises: [].} - ## Represents an arbitrary Nim reference value as YAML object. The object - ## may be represented as alias node if it is already present in the - ## ``SerializationContext``. - -proc representChild*(value: string, ts: TagStyle, c: SerializationContext): - RawYamlStream {.inline.} - ## Represents a Nim string. Supports nil strings. - -proc representChild*[O](value: O, ts: TagStyle, - c: SerializationContext): - RawYamlStream {.raises: [].} - ## Represents an arbitrary Nim object as YAML object. - -proc construct*[T](s: var YamlStream, target: var T) - {.raises: [YamlConstructionError, YamlStreamError].} - ## Constructs a Nim value from a YAML stream. - -proc load*[K](input: Stream, target: var K) - {.raises: [YamlConstructionError, IOError, YamlParserError].} - ## Loads a Nim value from a YAML character stream. - -proc represent*[T](value: T, ts: TagStyle = tsRootOnly, - a: AnchorStyle = asTidy): YamlStream {.raises: [].} - ## Represents a Nim value as ``YamlStream`` - -proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly, - anchorStyle: AnchorStyle = asTidy, - options: PresentationOptions = defaultPresentationOptions) - {.raises: [YamlPresenterJsonError, YamlPresenterOutputError].} - ## Dump a Nim value as YAML character stream. - -var - serializationTagLibrary* = initSerializationTagLibrary() ## \ - ## contains all local tags that are used for type serialization. Does - ## not contain any of the specific default tags for sequences or maps, - ## as those are not suited for Nim's static type system. - ## - ## Should not be modified manually. Will be extended by - ## `serializable <#serializable,stmt,stmt>`_. - -var - nextStaticTagId {.compileTime.} = 100.TagId ## \ - ## used for generating unique TagIds with ``setTagUri``. - registeredUris {.compileTime.} = newSeq[string]() ## \ - ## Since Table doesn't really work at compile time, we also store - ## registered URIs here to be able to generate a static compiler error - ## when the user tries to register an URI more than once. - -template setTagUri*(t: typedesc, uri: string): stmt = - ## Associate the given uri with a certain type. This uri is used as YAML tag - ## when loading and dumping values of this type. - when uri in registeredUris: - {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .} - const id {.genSym.} = nextStaticTagId - static: - registeredUris.add(uri) - nextStaticTagId = TagId(int(nextStaticTagId) + 1) - when nextStaticTagId == yFirstCustomTagId: - {.fatal: "Too many tags!".} - serializationTagLibrary.tags[uri] = id - proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = id - ## autogenerated - -template setTagUri*(t: typedesc, uri: string, idName: expr): stmt = - ## Like `setTagUri <#setTagUri.t,typedesc,string>`_, but lets - ## you choose a symbol for the `TagId <#TagId>`_ of the uri. This is only - ## necessary if you want to implement serialization / construction yourself. - when uri in registeredUris: - {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .} - const idName* = nextStaticTagId - static: - registeredUris.add(uri) - nextStaticTagId = TagId(int(nextStaticTagId) + 1) - serializationTagLibrary.tags[uri] = idName - proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName - ## autogenerated - -proc canBeImplicit(t: typedesc): bool {.compileTime.} = - let tDesc = getType(t) - if tDesc.kind != nnkObjectTy: return false - if tDesc[2].len != 1: return false - if tDesc[2][0].kind != nnkRecCase: return false - var foundEmptyBranch = false - for i in 1.. tDesc[2][0].len - 1: - case tDesc[2][0][i][1].len # branch contents - of 0: - if foundEmptyBranch: return false - else: foundEmptyBranch = true - of 1: discard - else: return false - return true - -template markAsImplicit*(t: typedesc): stmt = - ## Mark a variant object type as implicit. This requires the type to consist - ## of nothing but a case expression and each branch of the case expression - ## containing exactly one field - with the exception that one branch may - ## contain zero fields. - when canBeImplicit(t): - # this will be checked by means of compiles(implicitVariantObject(...)) - proc implicitVariantObject*(unused: t) = discard - else: - {. fatal: "This type cannot be marked as implicit" .} - -static: - # standard YAML tags used by serialization - registeredUris.add("!") - registeredUris.add("?") - registeredUris.add("tag:yaml.org,2002:str") - registeredUris.add("tag:yaml.org,2002:null") - registeredUris.add("tag:yaml.org,2002:bool") - registeredUris.add("tag:yaml.org,2002:float") - registeredUris.add("tag:yaml.org,2002:timestamp") - registeredUris.add("tag:yaml.org,2002:value") - registeredUris.add("tag:yaml.org,2002:binary") - # special tags used by serialization - registeredUris.add("!nim:field") - registeredUris.add("!nim:nil:string") - registeredUris.add("!nim:nil:seq") - -# tags for Nim's standard types -setTagUri(char, "!nim:system:char", yTagNimChar) -setTagUri(int8, "!nim:system:int8", yTagNimInt8) -setTagUri(int16, "!nim:system:int16", yTagNimInt16) -setTagUri(int32, "!nim:system:int32", yTagNimInt32) -setTagUri(int64, "!nim:system:int64", yTagNimInt64) -setTagUri(uint8, "!nim:system:uint8", yTagNimUInt8) -setTagUri(uint16, "!nim:system:uint16", yTagNimUInt16) -setTagUri(uint32, "!nim:system:uint32", yTagNimUInt32) -setTagUri(uint64, "!nim:system:uint64", yTagNimUInt64) -setTagUri(float32, "!nim:system:float32", yTagNimFloat32) -setTagUri(float64, "!nim:system:float64", yTagNimFloat64) - -# implementation - -include private.tagLibrary -include private.events -include private.json -include private.presenter -include private.hints -include private.fastparse -include private.streams -include private.serialization -include private.dom \ No newline at end of file +import yaml.dom, yaml.hints, yaml.parser, yaml.presenter, + yaml.serialization, yaml.stream, yaml.taglib, yaml.tojson +export yaml.dom, yaml.hints, yaml.parser, yaml.presenter, + yaml.serialization, yaml.stream, yaml.taglib, yaml.tojson \ No newline at end of file diff --git a/yaml/dom.nim b/yaml/dom.nim new file mode 100644 index 0000000..f4b3ee6 --- /dev/null +++ b/yaml/dom.nim @@ -0,0 +1,195 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## =============== +## Module yaml.dom +## =============== +## +## This is the DOM API, which enables you to load YAML into a tree-like +## structure. It can also dump the structure back to YAML. Formally, it +## represents the *Representation Graph* as defined in the YAML specification. +## +## The main interface of this API are ``loadDOM`` and ``dumpDOM``. The other +## exposed procs are low-level and useful if you want to load or generate parts +## of a ``YamlStream``. + +import tables, streams +import stream, taglib, serialization, ../private/internal, parser, + presenter + +type + YamlNodeKind* = enum + yScalar, yMapping, ySequence + + YamlNode* = ref YamlNodeObj not nil + ## Represents a node in a ``YamlDocument``. + + YamlNodeObj* = object + tag*: string + case kind*: YamlNodeKind + of yScalar: content*: string + of ySequence: children*: seq[YamlNode] + of yMapping: pairs*: seq[tuple[key, value: YamlNode]] + + YamlDocument* = object + ## Represents a YAML document. + root*: YamlNode + +proc newYamlNode*(content: string, tag: string = "?"): YamlNode = + YamlNode(kind: yScalar, content: content, tag: tag) + +proc newYamlNode*(children: openarray[YamlNode], tag: string = "?"): + YamlNode = + YamlNode(kind: ySequence, children: @children, tag: tag) + +proc newYamlNode*(pairs: openarray[tuple[key, value: YamlNode]], + tag: string = "?"): YamlNode = + YamlNode(kind: yMapping, pairs: @pairs, tag: tag) + +proc initYamlDoc*(root: YamlNode): YamlDocument = result.root = root + +proc composeNode(s: var YamlStream, tagLib: TagLibrary, + c: ConstructionContext): + YamlNode {.raises: [YamlStreamError, YamlConstructionError].} = + var start: YamlStreamEvent + shallowCopy(start, s.next()) + new(result) + try: + case start.kind + of yamlStartMap: + result.tag = tagLib.uri(start.mapTag) + result.kind = yMapping + result.pairs = newSeq[tuple[key, value: YamlNode]]() + while s.peek().kind != yamlEndMap: + let + key = composeNode(s, tagLib, c) + value = composeNode(s, tagLib, c) + result.pairs.add((key: key, value: value)) + discard s.next() + if start.mapAnchor != yAnchorNone: + yAssert(not c.refs.hasKey(start.mapAnchor)) + c.refs[start.mapAnchor] = cast[pointer](result) + of yamlStartSeq: + result.tag = tagLib.uri(start.seqTag) + result.kind = ySequence + result.children = newSeq[YamlNode]() + while s.peek().kind != yamlEndSeq: + result.children.add(composeNode(s, tagLib, c)) + if start.seqAnchor != yAnchorNone: + yAssert(not c.refs.hasKey(start.seqAnchor)) + c.refs[start.seqAnchor] = cast[pointer](result) + discard s.next() + of yamlScalar: + result.tag = tagLib.uri(start.scalarTag) + result.kind = yScalar + shallowCopy(result.content, start.scalarContent) + if start.scalarAnchor != yAnchorNone: + yAssert(not c.refs.hasKey(start.scalarAnchor)) + c.refs[start.scalarAnchor] = cast[pointer](result) + of yamlAlias: + result = cast[YamlNode](c.refs[start.aliasTarget]) + else: internalError("Malformed YamlStream") + except KeyError: + raise newException(YamlConstructionError, + "Wrong tag library: TagId missing") + +proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument + {.raises: [YamlStreamError, YamlConstructionError].} = + var context = newConstructionContext() + var n: YamlStreamEvent + shallowCopy(n, s.next()) + yAssert n.kind == yamlStartDoc + result.root = composeNode(s, tagLib, context) + n = s.next() + yAssert n.kind == yamlEndDoc + +proc loadDOM*(s: Stream | string): YamlDocument + {.raises: [IOError, YamlParserError, YamlConstructionError].} = + var + tagLib = initExtendedTagLibrary() + parser = newYamlParser(tagLib) + events = parser.parse(s) + try: result = compose(events, tagLib) + except YamlStreamError: + let e = getCurrentException() + if e.parent of YamlParserError: + raise (ref YamlParserError)(e.parent) + elif e.parent of IOError: + raise (ref IOError)(e.parent) + else: internalError("Unexpected exception: " & e.parent.repr) + +proc serializeNode(n: YamlNode, c: SerializationContext, a: AnchorStyle, + tagLib: TagLibrary) {.raises: [].}= + let p = cast[pointer](n) + if a != asNone and c.refs.hasKey(p): + if c.refs.getOrDefault(p) == yAnchorNone: + c.refs[p] = c.nextAnchorId + c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1) + c.put(aliasEvent(c.refs.getOrDefault(p))) + return + var + tagId: TagId + anchor: AnchorId + if a == asAlways: + c.refs[p] = c.nextAnchorId + c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1) + else: c.refs[p] = yAnchorNone + tagId = if tagLib.tags.hasKey(n.tag): tagLib.tags.getOrDefault(n.tag) else: + tagLib.registerUri(n.tag) + case a + of asNone: anchor = yAnchorNone + of asTidy: anchor = cast[AnchorId](n) + of asAlways: anchor = c.refs.getOrDefault(p) + + case n.kind + of yScalar: c.put(scalarEvent(n.content, tagId, anchor)) + of ySequence: + c.put(startSeqEvent(tagId, anchor)) + for item in n.children: + serializeNode(item, c, a, tagLib) + c.put(endSeqEvent()) + of yMapping: + c.put(startMapEvent(tagId, anchor)) + for i in n.pairs: + serializeNode(i.key, c, a, tagLib) + serializeNode(i.value, c, a, tagLib) + c.put(endMapEvent()) + +template processAnchoredEvent(target: untyped, c: SerializationContext): typed = + let anchorId = c.refs.getOrDefault(cast[pointer](target)) + if anchorId != yAnchorNone: target = anchorId + else: target = yAnchorNone + +proc serialize*(doc: YamlDocument, tagLib: TagLibrary, a: AnchorStyle = asTidy): + YamlStream {.raises: [].} = + var + bys = newBufferYamlStream() + c = newSerializationContext(a, proc(e: YamlStreamEvent) {.raises: [].} = + bys.put(e) + ) + c.put(startDocEvent()) + serializeNode(doc.root, c, a, tagLib) + c.put(endDocEvent()) + if a == asTidy: + for event in bys.mitems(): + case event.kind + of yamlScalar: processAnchoredEvent(event.scalarAnchor, c) + of yamlStartMap: processAnchoredEvent(event.mapAnchor, c) + of yamlStartSeq: processAnchoredEvent(event.seqAnchor, c) + else: discard + result = bys + +proc dumpDOM*(doc: YamlDocument, target: Stream, + anchorStyle: AnchorStyle = asTidy, + options: PresentationOptions = defaultPresentationOptions) + {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} = + ## Dump a YamlDocument as YAML character stream. + var + tagLib = initExtendedTagLibrary() + events = serialize(doc, tagLib, + if options.style == psJson: asNone else: anchorStyle) + present(events, target, tagLib, options) diff --git a/private/hints.nim b/yaml/hints.nim similarity index 71% rename from private/hints.nim rename to yaml/hints.nim index f3bf71e..0892787 100644 --- a/private/hints.nim +++ b/yaml/hints.nim @@ -1,10 +1,43 @@ # NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause +# (c) Copyright 2016 Felix Krause # # See the file "copying.txt", included in this # distribution, for details about the copyright. +## ================= +## Module yaml.hints +## ================= +## +## The hints API enables you to guess the type of YAML scalars. + +import macros +import ../private/internal + type + TypeHint* = enum + ## A type hint can be computed from scalar content and tells you what + ## NimYAML thinks the scalar's type is. It is generated by + ## `guessType <#guessType,string>`_ The first matching RegEx + ## in the following table will be the type hint of a scalar string. + ## + ## You can use it to determine the type of YAML scalars that have a '?' + ## non-specific tag, but using this feature is completely optional. + ## + ## ================== ========================= + ## Name RegEx + ## ================== ========================= + ## ``yTypeInteger`` ``0 | -? [1-9] [0-9]*`` + ## ``yTypeFloat`` ``-? [1-9] ( \. [0-9]* [1-9] )? ( e [-+] [1-9] [0-9]* )?`` + ## ``yTypeFloatInf`` ``-? \. (inf | Inf | INF)`` + ## ``yTypeFloatNaN`` ``-? \. (nan | NaN | NAN)`` + ## ``yTypeBoolTrue`` ``y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON`` + ## ``yTypeBoolFalse`` ``n|N|no|No|NO|false|False|FALSE|off|Off|OFF`` + ## ``yTypeNull`` ``~ | null | Null | NULL`` + ## ``yTypeUnknown`` ``*`` + ## ================== ========================= + yTypeInteger, yTypeFloat, yTypeFloatInf, yTypeFloatNaN, yTypeBoolTrue, + yTypeBoolFalse, yTypeNull, yTypeUnknown + YamlTypeHintState = enum ythInitial, ythF, ythFA, ythFAL, ythFALS, ythFALSE, @@ -28,11 +61,11 @@ type ythMinus, yth0, ythInt, ythDecimal, ythNumE, ythNumEPlusMinus, ythExponent -macro typeHintStateMachine(c: untyped, content: untyped): stmt = - assert content.kind == nnkStmtList +macro typeHintStateMachine(c: untyped, content: untyped): typed = + yAssert content.kind == nnkStmtList result = newNimNode(nnkCaseStmt, content).add(copyNimNode(c)) for branch in content.children: - assert branch.kind == nnkOfBranch + yAssert branch.kind == nnkOfBranch var charBranch = newNimNode(nnkOfBranch, branch) i = 0 @@ -42,14 +75,14 @@ macro typeHintStateMachine(c: untyped, content: untyped): stmt = charBranch.add(copyNimTree(branch[i])) inc(i) for rule in branch[i].children: - assert rule.kind == nnkInfix - assert ($rule[0].ident == "=>") + yAssert rule.kind == nnkInfix + yAssert $rule[0].ident == "=>" var stateBranch = newNimNode(nnkOfBranch, rule) case rule[1].kind of nnkBracket: for item in rule[1].children: stateBranch.add(item) of nnkIdent: stateBranch.add(rule[1]) - else: assert false + else: internalError("Invalid rule kind: " & $rule[1].kind) if rule[2].kind == nnkNilLit: stateBranch.add(newStmtList(newNimNode(nnkDiscardStmt).add( newEmptyNode()))) @@ -150,7 +183,9 @@ template advanceTypeHint(ch: char) {.dirty.} = ythTR => ythTRU of 'y', 'Y': ythInitial => ythY -proc guessType*(scalar: string): TypeHint = +proc guessType*(scalar: string): TypeHint {.raises: [].} = + ## Parse scalar string according to the RegEx table documented at + ## `TypeHint <#TypeHind>`_. var typeHintState: YamlTypeHintState = ythInitial for c in scalar: advanceTypeHint(c) case typeHintState @@ -161,4 +196,4 @@ proc guessType*(scalar: string): TypeHint = of ythDecimal, ythExponent: result = yTypeFloat of ythPointINF: result = yTypeFloatInf of ythPointNAN: result = yTypeFloatNaN - else: result = yTypeUnknown \ No newline at end of file + else: result = yTypeUnknown diff --git a/yaml/parser.nim b/yaml/parser.nim new file mode 100644 index 0000000..200cf28 --- /dev/null +++ b/yaml/parser.nim @@ -0,0 +1,1056 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## ================== +## Module yaml.parser +## ================== +## +## This is the low-level parser API. A ``YamlParser`` enables you to parse any +## non-nil string or Stream object as YAML character stream. + +import tables, strutils, macros, streams +import taglib, stream, ../private/lex, ../private/internal + +type + WarningCallback* = proc(line, column: int, lineContent: string, + message: string) + ## Callback for parser warnings. Currently, this callback may be called + ## on two occasions while parsing a YAML document stream: + ## + ## - If the version number in the ``%YAML`` directive does not match + ## ``1.2``. + ## - If there is an unknown directive encountered. + + YamlParser* = ref object + ## A parser object. Retains its ``TagLibrary`` across calls to + ## `parse <#parse,YamlParser,Stream>`_. Can be used + ## to access anchor names while parsing a YAML character stream, but + ## only until the document goes out of scope (i.e. until + ## ``yamlEndDocument`` is yielded). + tagLib: TagLibrary + callback: WarningCallback + anchors: Table[string, AnchorId] + + FastParseLevelKind = enum + fplUnknown, fplSequence, fplMapKey, fplMapValue, fplSinglePairKey, + fplSinglePairValue, fplDocument + + FastParseLevel = object + kind: FastParseLevelKind + indentation: int + + ParserContext = ref object of YamlStream + p: YamlParser + lex: YamlLexer + storedState: proc(s: YamlStream, e: var YamlStreamEvent): bool + atSequenceItem: bool + flowdepth: int + ancestry: seq[FastParseLevel] + level: FastParseLevel + tag: TagId + anchor: AnchorId + shorthands: Table[string, string] + nextAnchorId: AnchorId + newlines: int + explicitFlowKey: bool + plainScalarStart: tuple[line, column: int] + + LevelEndResult = enum + lerNothing, lerOne, lerAdditionalMapEnd + + YamlLoadingError* = object of Exception + ## Base class for all exceptions that may be raised during the process + ## of loading a YAML character stream. + line*: int ## line number (1-based) where the error was encountered + column*: int ## column number (1-based) where the error was encountered + lineContent*: string ## \ + ## content of the line where the error was encountered. Includes a + ## second line with a marker ``^`` at the position where the error + ## was encountered. + + YamlParserError* = object of YamlLoadingError + ## A parser error is raised if the character stream that is parsed is + ## not a valid YAML character stream. This stream cannot and will not be + ## parsed wholly nor partially and all events that have been emitted by + ## the YamlStream the parser provides should be discarded. + ## + ## A character stream is invalid YAML if and only if at least one of the + ## following conditions apply: + ## + ## - There are invalid characters in an element whose contents is + ## restricted to a limited set of characters. For example, there are + ## characters in a tag URI which are not valid URI characters. + ## - An element has invalid indentation. This can happen for example if + ## a block list element indicated by ``"- "`` is less indented than + ## the element in the previous line, but there is no block sequence + ## list open at the same indentation level. + ## - The YAML structure is invalid. For example, an explicit block map + ## indicated by ``"? "`` and ``": "`` may not suddenly have a block + ## sequence item (``"- "``) at the same indentation level. Another + ## possible violation is closing a flow style object with the wrong + ## closing character (``}``, ``]``) or not closing it at all. + ## - A custom tag shorthand is used that has not previously been + ## declared with a ``%TAG`` directive. + ## - Multiple tags or anchors are defined for the same node. + ## - An alias is used which does not map to any anchor that has + ## previously been declared in the same document. + ## - An alias has a tag or anchor associated with it. + ## + ## Some elements in this list are vague. For a detailed description of a + ## valid YAML character stream, see the YAML specification. + +proc newYamlParser*(tagLib: TagLibrary = initExtendedTagLibrary(), + callback: WarningCallback = nil): YamlParser = + ## Creates a YAML parser. if ``callback`` is not ``nil``, it will be called + ## whenever the parser yields a warning. + new(result) + result.tagLib = tagLib + result.callback = callback + +template debug(message: string) {.dirty.} = + when defined(yamlDebug): + try: styledWriteLine(stdout, fgBlue, message) + except IOError: discard + +proc generateError(c: ParserContext, message: string): + ref YamlParserError {.raises: [].} = + result = newException(YamlParserError, message) + (result.line, result.column) = c.lex.curStartPos + result.lineContent = c.lex.getTokenLine() + +proc illegalToken(c: ParserContext, expected: string = ""): + ref YamlParserError {.raises: [].} = + var msg = "Illegal token" + if expected.len > 0: msg.add(" (expected " & expected & ")") + msg.add(": " & $c.lex.cur) + result = c.generateError(msg) + +proc callCallback(c: ParserContext, msg: string) {.raises: [YamlParserError].} = + try: + if not isNil(c.p.callback): + c.p.callback(c.lex.curStartPos.line, c.lex.curStartPos.column, + c.lex.getTokenLine(), msg) + except: + var e = newException(YamlParserError, + "Warning callback raised exception: " & getCurrentExceptionMsg()) + e.parent = getCurrentException() + raise e + +proc initLevel(k: FastParseLevelKind): FastParseLevel {.raises: [], inline.} = + FastParseLevel(kind: k, indentation: UnknownIndentation) + +proc emptyScalar(c: ParserContext): YamlStreamEvent {.raises: [], inline.} = + result = scalarEvent("", c.tag, c.anchor) + c.tag = yTagQuestionMark + c.anchor = yAnchorNone + +proc currentScalar(c: ParserContext, e: var YamlStreamEvent) + {.raises: [], inline.} = + e = YamlStreamEvent(kind: yamlScalar, scalarTag: c.tag, + scalarAnchor: c.anchor) + shallowCopy(e.scalarContent, c.lex.buf) + c.lex.buf = cast[string not nil](newStringOfCap(256)) + c.tag = yTagQuestionMark + c.anchor = yAnchorNone + +proc objectStart(c: ParserContext, k: static[YamlStreamEventKind], + single: bool = false): YamlStreamEvent {.raises: [].} = + yAssert(c.level.kind == fplUnknown) + when k == yamlStartMap: + result = startMapEvent(c.tag, c.anchor) + if single: + debug("started single-pair map at " & + (if c.level.indentation == UnknownIndentation: + $c.lex.indentation else: $c.level.indentation)) + c.level.kind = fplSinglePairKey + else: + debug("started map at " & + (if c.level.indentation == UnknownIndentation: + $c.lex.indentation else: $c.level.indentation)) + c.level.kind = fplMapKey + else: + result = startSeqEvent(c.tag, c.anchor) + debug("started sequence at " & + (if c.level.indentation == UnknownIndentation: $c.lex.indentation else: + $c.level.indentation)) + c.level.kind = fplSequence + c.tag = yTagQuestionMark + c.anchor = yAnchorNone + if c.level.indentation == UnknownIndentation: + c.level.indentation = c.lex.indentation + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + +proc initDocValues(c: ParserContext) {.raises: [].} = + c.shorthands = initTable[string, string]() + c.p.anchors = initTable[string, AnchorId]() + c.shorthands["!"] = "!" + c.shorthands["!!"] = "tag:yaml.org,2002:" + c.nextAnchorId = 0.AnchorId + c.level = initLevel(fplUnknown) + c.tag = yTagQuestionMark + c.anchor = yAnchorNone + c.ancestry.add(FastParseLevel(kind: fplDocument, indentation: -1)) + +proc advance(c: ParserContext) {.inline.} = + try: c.lex.next() + except YamlLexerError: + let e = (ref YamlLexerError)(getCurrentException()) + let pe = newException(YamlParserError, e.msg) + pe.line = e.line + pe.column = e.column + pe.lineContent = e.lineContent + raise pe + +proc handleAnchor(c: ParserContext) {.raises: [YamlParserError].} = + if c.level.kind != fplUnknown: raise c.generateError("Unexpected token") + if c.anchor != yAnchorNone: + raise c.generateError("Only one anchor is allowed per node") + c.anchor = c.nextAnchorId + c.p.anchors[c.lex.buf] = c.anchor + c.nextAnchorId = AnchorId(int(c.nextAnchorId) + 1) + c.lex.buf.setLen(0) + c.advance() + +proc handleTagHandle(c: ParserContext) {.raises: [YamlParserError].} = + if c.level.kind != fplUnknown: raise c.generateError("Unexpected tag handle") + if c.tag != yTagQuestionMark: + raise c.generateError("Only one tag handle is allowed per node") + if c.lex.cur == ltTagHandle: + var tagUri = "" + try: + tagUri.add(c.shorthands[c.lex.buf[0..c.lex.shorthandEnd]]) + tagUri.add(c.lex.buf[c.lex.shorthandEnd + 1 .. ^1]) + except KeyError: + raise c.generateError( + "Undefined tag shorthand: " & c.lex.buf[0..c.lex.shorthandEnd]) + try: c.tag = c.p.tagLib.tags[tagUri] + except KeyError: c.tag = c.p.tagLib.registerUri(tagUri) + else: + try: c.tag = c.p.tagLib.tags[c.lex.buf] + except KeyError: c.tag = c.p.tagLib.registerUri(c.lex.buf) + c.lex.buf.setLen(0) + c.advance() + +proc handlePossibleMapStart(c: ParserContext, e: var YamlStreamEvent, + flow: bool = false, single: bool = false): bool = + result = false + if c.level.indentation == UnknownIndentation: + if c.lex.isImplicitKeyStart(): + e = c.objectStart(yamlStartMap, single) + result = true + c.level.indentation = c.lex.indentation + +proc handleMapKeyIndicator(c: ParserContext, e: var YamlStreamEvent): bool = + result = false + case c.level.kind + of fplUnknown: + e = c.objectStart(yamlStartMap) + result = true + of fplMapValue: + if c.level.indentation != c.lex.indentation: + raise c.generateError("Invalid p.indentation of map key indicator " & + "(expected" & $c.level.indentation & ", got " & $c.lex.indentation & + ")") + e = scalarEvent("", yTagQuestionMark, yAnchorNone) + result = true + c.level.kind = fplMapKey + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + of fplMapKey: + if c.level.indentation != c.lex.indentation: + raise c.generateError("Invalid p.indentation of map key indicator") + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + of fplSequence: + raise c.generateError("Unexpected map key indicator (expected '- ')") + of fplSinglePairKey, fplSinglePairValue, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + c.advance() + if c.lex.cur != ltIndentation: + # this enables the parser to properly parse compact structures, like + # ? - a + # - b + # and such. At the first `-`, the indentation must equal its level to be + # parsed properly. + c.lex.indentation = c.lex.curStartPos.column - 1 + +proc handleBlockSequenceIndicator(c: ParserContext, e: var YamlStreamEvent): + bool = + result = false + case c.level.kind + of fplUnknown: + e = c.objectStart(yamlStartSeq) + result = true + of fplSequence: + if c.level.indentation != c.lex.indentation: + raise c.generateError( + "Invalid p.indentation of block sequence indicator (expected " & + $c.level.indentation & ", got " & $c.lex.indentation & ")") + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + else: raise c.generateError("Illegal sequence item in map") + c.advance() + if c.lex.cur != ltIndentation: + # see comment in previous proc, this time with structures like + # - - a + # - b + c.lex.indentation = c.lex.curStartPos.column - 1 + +proc handleBlockItemStart(c: ParserContext, e: var YamlStreamEvent): bool = + result = false + case c.level.kind + of fplUnknown: + result = c.handlePossibleMapStart(e) + of fplSequence: + raise c.generateError( + "Unexpected token (expected block sequence indicator)") + of fplMapKey: + c.ancestry.add(c.level) + c.level = FastParseLevel(kind: fplUnknown, indentation: c.lex.indentation) + of fplMapValue: + e = emptyScalar(c) + result = true + c.level.kind = fplMapKey + c.ancestry.add(c.level) + c.level = FastParseLevel(kind: fplUnknown, indentation: c.lex.indentation) + of fplSinglePairKey, fplSinglePairValue, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + +proc handleFlowItemStart(c: ParserContext, e: var YamlStreamEvent): bool = + if c.level.kind == fplUnknown and + c.ancestry[c.ancestry.high].kind == fplSequence: + result = c.handlePossibleMapStart(e, true, true) + else: result = false + +proc handleFlowPlainScalar(c: ParserContext) = + while c.lex.cur in {ltScalarPart, ltEmptyLine}: + c.lex.newlines.inc() + c.advance() + c.lex.newlines = 0 + +proc lastTokenContext(s: YamlStream, line, column: var int, + lineContent: var string): bool = + let c = ParserContext(s) + line = c.lex.curStartPos.line + column = c.lex.curStartPos.column + lineContent = c.lex.getTokenLine(true) + result = true + +# --- macros for defining parser states --- + +template capitalize(s: string): string = + when declared(strutils.capitalizeAscii): strutils.capitalizeAscii(s) + else: strutils.capitalize(s) + +macro parserStates(names: varargs[untyped]): typed = + ## generates proc declaration for each state in list like this: + ## + ## proc name(s: YamlStream, e: var YamlStreamEvent): + ## bool {.raises: [YamlParserError].} + result = newStmtList() + for name in names: + let nameId = newIdentNode("state" & capitalize($name.ident)) + result.add(newProc(nameId, [ident("bool"), newIdentDefs(ident("s"), + ident("YamlStream")), newIdentDefs(ident("e"), newNimNode(nnkVarTy).add( + ident("YamlStreamEvent")))], newEmptyNode())) + result[0][4] = newNimNode(nnkPragma).add(newNimNode(nnkExprColonExpr).add( + ident("raises"), newNimNode(nnkBracket).add(ident("YamlParserError"), + ident("YamlLexerError")))) + +proc processStateAsgns(source, target: NimNode) {.compileTime.} = + ## copies children of source to target and replaces all assignments + ## `state = [name]` with the appropriate code for changing states. + for child in source.children: + if child.kind == nnkAsgn and child[0].kind == nnkIdent: + if $child[0].ident == "state": + assert child[1].kind == nnkIdent + var newNameId: NimNode + if child[1].kind == nnkIdent and $child[1].ident == "stored": + newNameId = newDotExpr(ident("c"), ident("storedState")) + else: + newNameId = + newIdentNode("state" & capitalize($child[1].ident)) + target.add(newAssignment(newDotExpr( + newIdentNode("s"), newIdentNode("nextImpl")), newNameId)) + continue + elif $child[0].ident == "stored": + assert child[1].kind == nnkIdent + let newNameId = + newIdentNode("state" & capitalize($child[1].ident)) + target.add(newAssignment(newDotExpr(newIdentNode("c"), + newIdentNode("storedState")), newNameId)) + continue + var processed = copyNimNode(child) + processStateAsgns(child, processed) + target.add(processed) + +macro parserState(name: untyped, impl: untyped): typed = + ## Creates a parser state. Every parser state is a proc with the signature + ## + ## proc(s: YamlStream, e: var YamlStreamEvent): + ## bool {.raises: [YamlParserError].} + ## + ## The proc name will be prefixed with "state" and the original name will be + ## capitalized, so a state "foo" will yield a proc named "stateFoo". + ## + ## Inside the proc, you have access to the ParserContext with the let variable + ## `c`. You can change the parser state by a assignment `state = [newState]`. + ## The [newState] must have been declared with states(...) previously. + let + nameStr = $name.ident + nameId = newIdentNode("state" & capitalize(nameStr)) + var procImpl = quote do: + debug("state: " & `nameStr`) + procImpl.add(newLetStmt(ident("c"), newCall("ParserContext", ident("s")))) + procImpl.add(newAssignment(newIdentNode("result"), newLit(false))) + assert impl.kind == nnkStmtList + processStateAsgns(impl, procImpl) + result = newProc(nameId, [ident("bool"), + newIdentDefs(ident("s"), ident("YamlStream")), newIdentDefs(ident("e"), + newNimNode(nnkVarTy).add(ident("YamlStreamEvent")))], procImpl) + +# --- parser states --- + +parserStates(initial, blockLineStart, blockObjectStart, blockAfterObject, + scalarEnd, plainScalarEnd, objectEnd, expectDocEnd, startDoc, + afterDocument, closeMoreIndentedLevels, afterPlainScalarYield, + emitEmptyScalar, tagHandle, anchor, alias, flow, leaveFlowMap, + leaveFlowSeq, flowAfterObject, leaveFlowSinglePairMap) + +proc closeEverything(c: ParserContext) = + c.lex.indentation = -1 + c.nextImpl = stateCloseMoreIndentedLevels + +proc endLevel(c: ParserContext, e: var YamlStreamEvent): + LevelEndResult = + result = lerOne + case c.level.kind + of fplSequence: + e = endSeqEvent() + of fplMapKey: + e = endMapEvent() + of fplMapValue, fplSinglePairValue: + e = emptyScalar(c) + c.level.kind = fplMapKey + result = lerAdditionalMapEnd + of fplUnknown: + if c.ancestry.len > 1: + e = emptyScalar(c) # don't yield scalar for empty doc + else: + result = lerNothing + of fplDocument: + e = endDocEvent() + of fplSinglePairKey: + internalError("Unexpected level kind: " & $c.level.kind) + +proc handleMapValueIndicator(c: ParserContext, e: var YamlStreamEvent): bool = + result = false + case c.level.kind + of fplUnknown: + if c.level.indentation == UnknownIndentation: + e = c.objectStart(yamlStartMap) + result = true + c.storedState = c.nextImpl + c.nextImpl = stateEmitEmptyScalar + else: + e = emptyScalar(c) + result = true + c.ancestry[c.ancestry.high].kind = fplMapValue + of fplMapKey: + if c.level.indentation != c.lex.indentation: + raise c.generateError("Invalid p.indentation of map key indicator") + e = scalarEvent("", yTagQuestionMark, yAnchorNone) + result = true + c.level.kind = fplMapValue + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + of fplMapValue: + if c.level.indentation != c.lex.indentation: + raise c.generateError("Invalid p.indentation of map key indicator") + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + of fplSequence: + raise c.generateError("Unexpected map value indicator (expected '- ')") + of fplSinglePairKey, fplSinglePairValue, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + c.advance() + if c.lex.cur != ltIndentation: + # see comment in handleMapKeyIndicator, this time with structures like + # a: - a + # - b + c.lex.indentation = c.lex.curStartPos.column - 1 + +template handleObjectEnd(c: ParserContext, mayHaveEmptyValue: bool = false): + bool = + var result = false + c.level = c.ancestry.pop() + when mayHaveEmptyValue: + if c.level.kind == fplSinglePairValue: + result = true + c.level = c.ancestry.pop() + case c.level.kind + of fplMapKey: c.level.kind = fplMapValue + of fplSinglePairKey: c.level.kind = fplSinglePairValue + of fplMapValue: c.level.kind = fplMapKey + of fplSequence, fplDocument: discard + of fplUnknown, fplSinglePairValue: + internalError("Unexpected level kind: " & $c.level.kind) + result + +proc leaveFlowLevel(c: ParserContext, e: var YamlStreamEvent): bool = + c.flowdepth.dec() + result = (c.endLevel(e) == lerOne) # lerAdditionalMapEnd cannot happen + if c.flowdepth == 0: + c.lex.setFlow(false) + c.storedState = stateBlockAfterObject + else: + c.storedState = stateFlowAfterObject + c.nextImpl = stateObjectEnd + c.advance() + +parserState initial: + case c.lex.cur + of ltYamlDirective: + c.advance() + assert c.lex.cur == ltYamlVersion, $c.lex.cur + if c.lex.buf != "1.2": + c.callCallback("Version is not 1.2, but " & c.lex.buf) + c.lex.buf.setLen(0) + c.advance() + of ltTagDirective: + c.advance() + assert c.lex.cur == ltTagShorthand + var tagShorthand: string + shallowCopy(tagShorthand, c.lex.buf) + c.lex.buf = "" + c.advance() + assert c.lex.cur == ltTagUri + c.shorthands[tagShorthand] = c.lex.buf + c.lex.buf.setLen(0) + c.advance() + of ltUnknownDirective: + c.callCallback("Unknown directive: " & c.lex.buf) + c.lex.buf.setLen(0) + c.advance() + if c.lex.cur == ltUnknownDirectiveParams: + c.lex.buf.setLen(0) + c.advance() + of ltIndentation: + e = startDocEvent() + result = true + state = blockObjectStart + of ltStreamEnd: c.isFinished = true + of ltDirectivesEnd: + e = startDocEvent() + result = true + c.advance() + state = blockObjectStart + of ltDocumentEnd: + c.advance() + state = afterDocument + else: internalError("Unexpected lexer token: " & $c.lex.cur) + +parserState blockLineStart: + case c.lex.cur + of ltIndentation: c.advance() + of ltEmptyLine: c.advance() + of ltStreamEnd: + c.closeEverything() + stored = afterDocument + else: + if c.lex.indentation <= c.ancestry[^1].indentation: + state = closeMoreIndentedLevels + stored = blockObjectStart + else: + state = blockObjectStart + +parserState blockObjectStart: + case c.lex.cur + of ltEmptyLine: c.advance() + of ltIndentation: + c.advance() + c.level.indentation = UnknownIndentation + state = blockLineStart + of ltDirectivesEnd: + c.closeEverything() + stored = startDoc + of ltDocumentEnd: + c.advance() + c.closeEverything() + stored = afterDocument + of ltMapKeyInd: + result = c.handleMapKeyIndicator(e) + of ltMapValInd: + result = c.handleMapValueIndicator(e) + of ltQuotedScalar: + result = c.handleBlockItemStart(e) + c.advance() + state = scalarEnd + of ltBlockScalarHeader: + c.lex.indentation = c.ancestry[^1].indentation + c.advance() + assert c.lex.cur == ltBlockScalar + if c.level.indentation == UnknownIndentation: + c.level.indentation = c.lex.indentation + c.advance() + state = scalarEnd + of ltScalarPart: + result = c.handleBlockItemStart(e) + c.plainScalarStart = c.lex.curStartPos + while true: + c.advance() + case c.lex.cur + of ltIndentation: + if c.lex.indentation <= c.ancestry[^1].indentation: break + c.lex.newlines.inc() + of ltScalarPart: discard + of ltEmptyLine: c.lex.newlines.inc() + else: break + c.lex.newlines = 0 + state = plainScalarEnd + stored = blockAfterObject + of ltSeqItemInd: + result = c.handleBlockSequenceIndicator(e) + of ltTagHandle, ltLiteralTag: + result = c.handleBlockItemStart(e) + state = tagHandle + stored = blockObjectStart + of ltAnchor: + result = c.handleBlockItemStart(e) + state = anchor + stored = blockObjectStart + of ltAlias: + result = c.handleBlockItemStart(e) + state = alias + stored = blockAfterObject + of ltBraceOpen, ltBracketOpen: + result = c.handleBlockItemStart(e) + c.lex.setFlow(true) + state = flow + of ltStreamEnd: + c.closeEverything() + stored = afterDocument + else: + raise c.generateError("Unexpected token: " & $c.lex.cur) + +parserState scalarEnd: + if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark + c.currentScalar(e) + result = true + state = objectEnd + stored = blockAfterObject + +parserState plainScalarEnd: + c.currentScalar(e) + result = true + c.lastTokenContextImpl = proc(s: YamlStream, line, column: var int, + lineContent: var string): bool {.raises: [].} = + let c = ParserContext(s) + (line, column) = c.plainScalarStart + lineContent = c.lex.getTokenLine(c.plainScalarStart, true) + result = true + state = afterPlainScalarYield + stored = blockAfterObject + +parserState afterPlainScalarYield: + c.lastTokenContextImpl = lastTokenContext + state = objectEnd + +parserState blockAfterObject: + case c.lex.cur + of ltIndentation, ltEmptyLine: + c.advance() + state = blockLineStart + of ltMapValInd: + case c.level.kind + of fplUnknown: + e = c.objectStart(yamlStartMap) + result = true + of fplMapKey: + e = scalarEvent("", yTagQuestionMark, yAnchorNone) + result = true + c.level.kind = fplMapValue + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + of fplMapValue: + c.level.kind = fplMapValue + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + of fplSequence: raise c.illegalToken("sequence item") + of fplSinglePairKey, fplSinglePairValue, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + c.advance() + state = blockObjectStart + of ltDirectivesEnd: + c.closeEverything() + stored = startDoc + of ltStreamEnd: + c.closeEverything() + stored = afterDocument + else: raise c.illegalToken("':', comment or line end") + +parserState objectEnd: + if c.handleObjectEnd(true): + e = endMapEvent() + result = true + if c.level.kind == fplDocument: state = expectDocEnd + else: state = stored + +parserState expectDocEnd: + case c.lex.cur + of ltIndentation, ltEmptyLine: c.advance() + of ltDirectivesEnd: + e = endDocEvent() + result = true + state = startDoc + c.ancestry.setLen(0) + of ltDocumentEnd: + e = endDocEvent() + result = true + state = afterDocument + c.advance() + of ltStreamEnd: + e = endDocEvent() + result = true + c.isFinished = true + else: + raise c.generateError("Unexpected token (expected document end): " & + $c.lex.cur) + +parserState startDoc: + c.initDocValues() + e = startDocEvent() + result = true + c.advance() + state = blockObjectStart + +parserState afterDocument: + case c.lex.cur + of ltStreamEnd: c.isFinished = true + of ltEmptyLine: c.advance() + else: + c.initDocValues() + state = initial + +parserState closeMoreIndentedLevels: + if c.ancestry.len > 0: + let parent = c.ancestry[c.ancestry.high] + if parent.indentation >= c.lex.indentation: + if c.lex.cur == ltSeqItemInd: + if (c.lex.indentation == c.level.indentation and + c.level.kind == fplSequence) or + (c.lex.indentation == parent.indentation and + c.level.kind == fplUnknown and parent.kind != fplSequence): + state = stored + debug("Not closing because sequence indicator") + return false + debug("Closing because parent.indentation (" & $parent.indentation & + ") >= indentation(" & $c.lex.indentation & ")") + case c.endLevel(e) + of lerNothing: discard + of lerOne: result = true + of lerAdditionalMapEnd: return true + discard c.handleObjectEnd(false) + return result + debug("Not closing level because parent.indentation (" & + $parent.indentation & ") < indentation(" & $c.lex.indentation & + ")") + if c.level.kind == fplDocument: state = expectDocEnd + else: state = stored + elif c.lex.indentation == c.level.indentation: + debug("Closing document") + let res = c.endLevel(e) + yAssert(res == lerOne) + result = true + state = stored + else: + state = stored + +parserState emitEmptyScalar: + e = scalarEvent("", yTagQuestionMark, yAnchorNone) + result = true + state = stored + +parserState tagHandle: + c.handleTagHandle() + state = stored + +parserState anchor: + c.handleAnchor() + state = stored + +parserState alias: + if c.level.kind != fplUnknown: raise c.generateError("Unexpected token") + if c.anchor != yAnchorNone or c.tag != yTagQuestionMark: + raise c.generateError("Alias may not have anchor or tag") + var id: AnchorId + try: id = c.p.anchors[c.lex.buf] + except KeyError: raise c.generateError("Unknown anchor") + c.lex.buf.setLen(0) + e = aliasEvent(id) + c.advance() + result = true + state = objectEnd + +parserState flow: + case c.lex.cur + of ltBraceOpen: + if c.handleFlowItemStart(e): return true + e = c.objectStart(yamlStartMap) + result = true + c.flowdepth.inc() + c.explicitFlowKey = false + c.advance() + of ltBracketOpen: + if c.handleFlowItemStart(e): return true + e = c.objectStart(yamlStartSeq) + result = true + c.flowdepth.inc() + c.advance() + of ltBraceClose: + yAssert(c.level.kind == fplUnknown) + c.level = c.ancestry.pop() + state = leaveFlowMap + of ltBracketClose: + yAssert(c.level.kind == fplUnknown) + c.level = c.ancestry.pop() + state = leaveFlowSeq + of ltComma: + yAssert(c.level.kind == fplUnknown) + c.level = c.ancestry.pop() + case c.level.kind + of fplSequence: + e = c.emptyScalar() + result = true + of fplMapValue: + e = c.emptyScalar() + result = true + c.level.kind = fplMapKey + c.explicitFlowKey = false + of fplMapKey: + e = c.emptyScalar() + c.level.kind = fplMapValue + return true + of fplSinglePairValue: + e = c.emptyScalar() + result = true + c.level = c.ancestry.pop() + state = leaveFlowSinglePairMap + stored = flow + of fplUnknown, fplSinglePairKey, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + c.advance() + of ltMapValInd: + c.level = c.ancestry.pop() + case c.level.kind + of fplSequence: + e = startMapEvent(c.tag, c.anchor) + result = true + debug("started single-pair map at " & + (if c.level.indentation == UnknownIndentation: + $c.lex.indentation else: $c.level.indentation)) + c.tag = yTagQuestionMark + c.anchor = yAnchorNone + if c.level.indentation == UnknownIndentation: + c.level.indentation = c.lex.indentation + c.ancestry.add(c.level) + c.level = initLevel(fplSinglePairKey) + of fplMapValue, fplSinglePairValue: + raise c.generateError("Unexpected token (expected ',')") + of fplMapKey: + e = c.emptyScalar() + result = true + c.level.kind = fplMapValue + of fplSinglePairKey: + e = c.emptyScalar() + result = true + c.level.kind = fplSinglePairValue + of fplUnknown, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + if c.level.kind != fplSinglePairKey: c.advance() + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + of ltQuotedScalar: + if c.handleFlowItemStart(e): return true + if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark + c.currentScalar(e) + result = true + state = objectEnd + stored = flowAfterObject + c.advance() + of ltTagHandle, ltLiteralTag: + if c.handleFlowItemStart(e): return true + c.handleTagHandle() + of ltAnchor: + if c.handleFlowItemStart(e): return true + c.handleAnchor() + of ltAlias: + state = alias + stored = flowAfterObject + of ltMapKeyInd: + if c.explicitFlowKey: + raise c.generateError("Duplicate '?' in flow mapping") + elif c.level.kind == fplUnknown: + case c.ancestry[c.ancestry.high].kind + of fplMapKey, fplMapValue, fplDocument: discard + of fplSequence: + e = c.objectStart(yamlStartMap, true) + result = true + else: + raise c.generateError("Unexpected token") + c.explicitFlowKey = true + c.advance() + of ltScalarPart: + if c.handleFlowItemStart(e): return true + c.handleFlowPlainScalar() + c.currentScalar(e) + result = true + state = objectEnd + stored = flowAfterObject + else: + raise c.generateError("Unexpected toked: " & $c.lex.cur) + +parserState leaveFlowMap: + case c.level.kind + of fplMapValue: + e = c.emptyScalar() + c.level.kind = fplMapKey + return true + of fplMapKey: + if c.tag != yTagQuestionMark or c.anchor != yAnchorNone or + c.explicitFlowKey: + e = c.emptyScalar() + c.level.kind = fplMapValue + c.explicitFlowKey = false + return true + of fplSequence: + raise c.generateError("Unexpected token (expected ']')") + of fplSinglePairValue: + raise c.generateError("Unexpected token (expected ']')") + of fplUnknown, fplSinglePairKey, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + result = c.leaveFlowLevel(e) + +parserState leaveFlowSeq: + case c.level.kind + of fplSequence: + if c.tag != yTagQuestionMark or c.anchor != yAnchorNone: + e = c.emptyScalar() + return true + of fplSinglePairValue: + e = c.emptyScalar() + c.level = c.ancestry.pop() + state = leaveFlowSinglePairMap + stored = leaveFlowSeq + return true + of fplMapKey, fplMapValue: + raise c.generateError("Unexpected token (expected '}')") + of fplUnknown, fplSinglePairKey, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + result = c.leaveFlowLevel(e) + +parserState leaveFlowSinglePairMap: + e = endMapEvent() + result = true + state = stored + +parserState flowAfterObject: + case c.lex.cur + of ltBracketClose: + case c.level.kind + of fplSequence: discard + of fplMapKey, fplMapValue: + raise c.generateError("Unexpected token (expected '}')") + of fplSinglePairValue: + c.level = c.ancestry.pop() + yAssert(c.level.kind == fplSequence) + e = endMapEvent() + return true + of fplUnknown, fplSinglePairKey, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + result = c.leaveFlowLevel(e) + of ltBraceClose: + case c.level.kind + of fplMapKey, fplMapValue: discard + of fplSequence, fplSinglePairValue: + raise c.generateError("Unexpected token (expected ']')") + of fplUnknown, fplSinglePairKey, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + result = c.leaveFlowLevel(e) + of ltComma: + case c.level.kind + of fplSequence: discard + of fplMapValue: + e = scalarEvent("", yTagQuestionMark, yAnchorNone) + result = true + c.level.kind = fplMapKey + c.explicitFlowKey = false + of fplSinglePairValue: + c.level = c.ancestry.pop() + yAssert(c.level.kind == fplSequence) + e = endMapEvent() + result = true + of fplMapKey: c.explicitFlowKey = false + of fplUnknown, fplSinglePairKey, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + state = flow + c.advance() + of ltMapValInd: + case c.level.kind + of fplSequence, fplMapKey: + raise c.generateError("Unexpected token (expected ',')") + of fplMapValue, fplSinglePairValue: discard + of fplUnknown, fplSinglePairKey, fplDocument: + internalError("Unexpected level kind: " & $c.level.kind) + c.ancestry.add(c.level) + c.level = initLevel(fplUnknown) + state = flow + c.advance() + of ltStreamEnd: + raise c.generateError("Unclosed flow content") + else: + raise c.generateError("Unexpected content (expected flow indicator)") + +# --- parser initialization --- + +proc init(c: ParserContext, p: YamlParser) = + c.basicInit(lastTokenContext) + c.p = p + c.ancestry = newSeq[FastParseLevel]() + c.initDocValues() + c.flowdepth = 0 + c.nextImpl = stateInitial + c.explicitFlowKey = false + c.advance() + +proc parse*(p: YamlParser, s: Stream): YamlStream + {.raises: [YamlParserError].} = + ## Parse the given stream as YAML character stream. + let c = new(ParserContext) + try: c.lex = newYamlLexer(s) + except: + let e = newException(YamlParserError, + "Error while opening stream: " & getCurrentExceptionMsg()) + e.parent = getCurrentException() + e.line = 1 + e.column = 1 + e.lineContent = "" + raise e + c.init(p) + result = c + +proc parse*(p: YamlParser, str: string): YamlStream + {.raises: [YamlParserError].} = + ## Parse the given string as YAML character stream. + let c = new(ParserContext) + c.lex = newYamlLexer(str) + c.init(p) + result = c diff --git a/private/presenter.nim b/yaml/presenter.nim similarity index 57% rename from private/presenter.nim rename to yaml/presenter.nim index fd54732..f5b9537 100644 --- a/private/presenter.nim +++ b/yaml/presenter.nim @@ -1,23 +1,126 @@ # NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause +# (c) Copyright 2016 Felix Krause # # See the file "copying.txt", included in this # distribution, for details about the copyright. +## ===================== +## Module yaml.presenter +## ===================== +## +## This is the presenter API, used for generating YAML character streams. + +import streams, queues, strutils +import taglib, stream, ../private/internal, hints, parser, stream + type + PresentationStyle* = enum + ## Different styles for YAML character stream output. + ## + ## - ``ypsMinimal``: Single-line flow-only output which tries to + ## use as few characters as possible. + ## - ``ypsCanonical``: Canonical YAML output. Writes all tags except + ## for the non-specific tags ``?`` and ``!``, uses flow style, quotes + ## all string scalars. + ## - ``ypsDefault``: Tries to be as human-readable as possible. Uses + ## block style by default, but tries to condense mappings and + ## sequences which only contain scalar nodes into a single line using + ## flow style. + ## - ``ypsJson``: Omits the ``%YAML`` directive and the ``---`` + ## marker. Uses flow style. Flattens anchors and aliases, omits tags. + ## Output will be parseable as JSON. ``YamlStream`` to dump may only + ## contain one document. + ## - ``ypsBlockOnly``: Formats all output in block style, does not use + ## flow style at all. + psMinimal, psCanonical, psDefault, psJson, psBlockOnly + + TagStyle* = enum + ## Whether object should be serialized with explicit tags. + ## + ## - ``tsNone``: No tags will be outputted unless necessary. + ## - ``tsRootOnly``: A tag will only be outputted for the root tag and + ## where necessary. + ## - ``tsAll``: Tags will be outputted for every object. + tsNone, tsRootOnly, tsAll + + AnchorStyle* = enum + ## How ref object should be serialized. + ## + ## - ``asNone``: No anchors will be outputted. Values present at + ## multiple places in the content that should be serialized will be + ## fully serialized at every occurence. If the content is cyclic, this + ## will lead to an endless loop! + ## - ``asTidy``: Anchors will only be generated for objects that + ## actually occur more than once in the content to be serialized. + ## This is a bit slower and needs more memory than ``asAlways``. + ## - ``asAlways``: Achors will be generated for every ref object in the + ## content to be serialized, regardless of whether the object is + ## referenced again afterwards + asNone, asTidy, asAlways + + NewLineStyle* = enum + ## What kind of newline sequence is used when presenting. + ## + ## - ``nlLF``: Use a single linefeed char as newline. + ## - ``nlCRLF``: Use a sequence of carriage return and linefeed as + ## newline. + ## - ``nlOSDefault``: Use the target operation system's default newline + ## sequence (CRLF on Windows, LF everywhere else). + nlLF, nlCRLF, nlOSDefault + + OutputYamlVersion* = enum + ## Specify which YAML version number the presenter shall emit. The + ## presenter will always emit content that is valid YAML 1.1, but by + ## default will write a directive ``%YAML 1.2``. For compatibility with + ## other YAML implementations, it is possible to change this here. + ## + ## It is also possible to specify that the presenter shall not emit any + ## YAML version. The generated content is then guaranteed to be valid + ## YAML 1.1 and 1.2 (but not 1.0 or any newer YAML version). + ov1_2, ov1_1, ovNone + + PresentationOptions* = object + ## Options for generating a YAML character stream + style*: PresentationStyle + indentationStep*: int + newlines*: NewLineStyle + outputVersion*: OutputYamlVersion + + YamlPresenterJsonError* = object of Exception + ## Exception that may be raised by the YAML presenter when it is + ## instructed to output JSON, but is unable to do so. This may occur if: + ## + ## - The given `YamlStream <#YamlStream>`_ contains a map which has any + ## non-scalar type as key. + ## - Any float scalar bears a ``NaN`` or positive/negative infinity value + + YamlPresenterOutputError* = object of Exception + ## Exception that may be raised by the YAML presenter. This occurs if + ## writing character data to the output stream raises any exception. + ## The error that has occurred is available from ``parent``. + DumperState = enum dBlockExplicitMapKey, dBlockImplicitMapKey, dBlockMapValue, dBlockInlineMap, dBlockSequenceItem, dFlowImplicitMapKey, dFlowMapValue, dFlowExplicitMapKey, dFlowSequenceItem, dFlowMapStart, dFlowSequenceStart - + ScalarStyle = enum sLiteral, sFolded, sPlain, sDoubleQuoted + PresenterTarget = Stream | ptr[string] + +const + defaultPresentationOptions* = + PresentationOptions(style: psDefault, indentationStep: 2, + newlines: nlOSDefault) + proc defineOptions*(style: PresentationStyle = psDefault, indentationStep: int = 2, newlines: NewLineStyle = nlOSDefault, outputVersion: OutputYamlVersion = ov1_2): - PresentationOptions = + PresentationOptions {.raises: [].} = + ## Define a set of options for presentation. Convenience proc that requires + ## you to only set those values that should not equal the default. PresentationOptions(style: style, indentationStep: indentationStep, newlines: newlines, outputVersion: outputVersion) @@ -94,110 +197,118 @@ proc inspect(scalar: string, indentation: int, elif canUseFolded: result = sFolded elif canUsePlain: result = sPlain else: result = sDoubleQuoted - -proc writeDoubleQuoted(scalar: string, s: Stream, indentation: int, + +template append(target: Stream, val: string | char) = + target.write(val) + +template append(target: ptr[string], val: string | char) = + target[].add(val) + +proc writeDoubleQuoted(scalar: string, s: PresenterTarget, indentation: int, newline: string) {.raises: [YamlPresenterOutputError].} = var curPos = indentation try: - s.write('"') + s.append('"') curPos.inc() for c in scalar: if curPos == 79: - s.write('\\') - s.write(newline) - s.write(repeat(' ', indentation)) + s.append('\\') + s.append(newline) + s.append(repeat(' ', indentation)) curPos = indentation if c == ' ': - s.write('\\') + s.append('\\') curPos.inc() case c of '"': - s.write("\\\"") + s.append("\\\"") curPos.inc(2) of '\l': - s.write("\\n") + s.append("\\n") curPos.inc(2) of '\t': - s.write("\\t") + s.append("\\t") curPos.inc(2) of '\\': - s.write("\\\\") + s.append("\\\\") curPos.inc(2) else: if ord(c) < 32: - s.write("\\x" & toHex(ord(c), 2)) + s.append("\\x" & toHex(ord(c), 2)) curPos.inc(4) else: - s.write(c) + s.append(c) curPos.inc() - s.write('"') + s.append('"') except: var e = newException(YamlPresenterOutputError, "Error while writing to output stream") e.parent = getCurrentException() raise e -proc writeDoubleQuotedJson(scalar: string, s: Stream) +proc writeDoubleQuotedJson(scalar: string, s: PresenterTarget) {.raises: [YamlPresenterOutputError].} = try: - s.write('"') + s.append('"') for c in scalar: case c - of '"': s.write("\\\"") - of '\\': s.write("\\\\") - of '\l': s.write("\\n") - of '\t': s.write("\\t") - of '\f': s.write("\\f") - of '\b': s.write("\\b") + of '"': s.append("\\\"") + of '\\': s.append("\\\\") + of '\l': s.append("\\n") + of '\t': s.append("\\t") + of '\f': s.append("\\f") + of '\b': s.append("\\b") else: - if ord(c) < 32: s.write("\\u" & toHex(ord(c), 4)) else: s.write(c) - s.write('"') + if ord(c) < 32: s.append("\\u" & toHex(ord(c), 4)) else: s.append(c) + s.append('"') except: var e = newException(YamlPresenterOutputError, "Error while writing to output stream") e.parent = getCurrentException() raise e -proc writeLiteral(scalar: string, indentation, indentStep: int, s: Stream, - lines: seq[tuple[start, finish: int]], newline: string) +proc writeLiteral(scalar: string, indentation, indentStep: int, + s: PresenterTarget, lines: seq[tuple[start, finish: int]], + newline: string) {.raises: [YamlPresenterOutputError].} = try: - s.write('|') - if scalar[^1] != '\l': s.write('-') - if scalar[0] in [' ', '\t']: s.write($indentStep) + s.append('|') + if scalar[^1] != '\l': s.append('-') + if scalar[0] in [' ', '\t']: s.append($indentStep) for line in lines: - s.write(newline) - s.write(repeat(' ', indentation + indentStep)) + s.append(newline) + s.append(repeat(' ', indentation + indentStep)) if line.finish >= line.start: - s.write(scalar[line.start .. line.finish]) + s.append(scalar[line.start .. line.finish]) except: var e = newException(YamlPresenterOutputError, "Error while writing to output stream") e.parent = getCurrentException() raise e -proc writeFolded(scalar: string, indentation, indentStep: int, s: Stream, - words: seq[tuple[start, finish: int]], newline: string) +proc writeFolded(scalar: string, indentation, indentStep: int, + s: PresenterTarget, words: seq[tuple[start, finish: int]], + newline: string) {.raises: [YamlPresenterOutputError].} = try: - s.write(">") - if scalar[^1] != '\l': s.write('-') - if scalar[0] in [' ', '\t']: s.write($indentStep) + s.append(">") + if scalar[^1] != '\l': s.append('-') + if scalar[0] in [' ', '\t']: s.append($indentStep) var curPos = 80 for word in words: if word.start > 0 and scalar[word.start - 1] == '\l': - s.write(newline & newline) - s.write(repeat(' ', indentation + indentStep)) + s.append(newline & newline) + s.append(repeat(' ', indentation + indentStep)) curPos = indentation + indentStep elif curPos + (word.finish - word.start) > 80: - s.write(newline) - s.write(repeat(' ', indentation + indentStep)) + s.append(newline) + s.append(repeat(' ', indentation + indentStep)) curPos = indentation + indentStep else: - s.write(' ') + s.append(' ') curPos.inc() - s.write(scalar[word.start .. word.finish]) + s.append(scalar[word.start .. word.finish]) curPos += word.finish - word.start + 1 except: var e = newException(YamlPresenterOutputError, @@ -205,82 +316,82 @@ proc writeFolded(scalar: string, indentation, indentStep: int, s: Stream, e.parent = getCurrentException() raise e -template safeWrite(s: string or char) {.dirty.} = - try: target.write(s) +template safeWrite(target: PresenterTarget, s: string or char) = + try: target.append(s) except: var e = newException(YamlPresenterOutputError, "") e.parent = getCurrentException() raise e -proc startItem(target: Stream, style: PresentationStyle, indentation: int, - state: var DumperState, isObject: bool, newline: string) - {.raises: [YamlPresenterOutputError].} = +proc startItem(target: PresenterTarget, style: PresentationStyle, + indentation: int, state: var DumperState, isObject: bool, + newline: string) {.raises: [YamlPresenterOutputError].} = try: case state of dBlockMapValue: - target.write(newline) - target.write(repeat(' ', indentation)) + target.append(newline) + target.append(repeat(' ', indentation)) if isObject or style == psCanonical: - target.write("? ") + target.append("? ") state = dBlockExplicitMapKey else: state = dBlockImplicitMapKey of dBlockInlineMap: state = dBlockImplicitMapKey of dBlockExplicitMapKey: - target.write(newline) - target.write(repeat(' ', indentation)) - target.write(": ") + target.append(newline) + target.append(repeat(' ', indentation)) + target.append(": ") state = dBlockMapValue of dBlockImplicitMapKey: - target.write(": ") + target.append(": ") state = dBlockMapValue of dFlowExplicitMapKey: if style != psMinimal: - target.write(newline) - target.write(repeat(' ', indentation)) - target.write(": ") + target.append(newline) + target.append(repeat(' ', indentation)) + target.append(": ") state = dFlowMapValue of dFlowMapValue: if (isObject and style != psMinimal) or style in [psJson, psCanonical]: - target.write(',' & newline & repeat(' ', indentation)) + target.append(',' & newline & repeat(' ', indentation)) if style == psJson: state = dFlowImplicitMapKey else: - target.write("? ") + target.append("? ") state = dFlowExplicitMapKey elif isObject and style == psMinimal: - target.write(", ? ") + target.append(", ? ") state = dFlowExplicitMapKey else: - target.write(", ") + target.append(", ") state = dFlowImplicitMapKey of dFlowMapStart: if (isObject and style != psMinimal) or style in [psJson, psCanonical]: - target.write(newline & repeat(' ', indentation)) + target.append(newline & repeat(' ', indentation)) if style == psJson: state = dFlowImplicitMapKey else: - target.write("? ") + target.append("? ") state = dFlowExplicitMapKey else: state = dFlowImplicitMapKey of dFlowImplicitMapKey: - target.write(": ") + target.append(": ") state = dFlowMapValue of dBlockSequenceItem: - target.write(newline) - target.write(repeat(' ', indentation)) - target.write("- ") + target.append(newline) + target.append(repeat(' ', indentation)) + target.append("- ") of dFlowSequenceStart: case style of psMinimal, psDefault: discard of psCanonical, psJson: - target.write(newline) - target.write(repeat(' ', indentation)) + target.append(newline) + target.append(repeat(' ', indentation)) of psBlockOnly: discard # can never happen state = dFlowSequenceItem of dFlowSequenceItem: case style - of psMinimal, psDefault: target.write(", ") + of psMinimal, psDefault: target.append(", ") of psCanonical, psJson: - target.write(',' & newline) - target.write(repeat(' ', indentation)) + target.append(',' & newline) + target.append(repeat(' ', indentation)) of psBlockOnly: discard # can never happen except: var e = newException(YamlPresenterOutputError, "") @@ -296,26 +407,27 @@ proc anchorName(a: AnchorId): string {.raises: [].} = else: result.add(char(j + ord('0') - 26)) i -= 36 -proc writeTagAndAnchor(target: Stream, tag: TagId, tagLib: TagLibrary, +proc writeTagAndAnchor(target: PresenterTarget, tag: TagId, + tagLib: TagLibrary, anchor: AnchorId) {.raises:[YamlPresenterOutputError].} = try: if tag notin [yTagQuestionMark, yTagExclamationMark]: let tagUri = tagLib.uri(tag) if tagUri.startsWith(tagLib.secondaryPrefix): - target.write("!!") - target.write(tagUri[18..tagUri.high]) - target.write(' ') + target.append("!!") + target.append(tagUri[18..tagUri.high]) + target.append(' ') elif tagUri.startsWith("!"): - target.write(tagUri) - target.write(' ') + target.append(tagUri) + target.append(' ') else: - target.write("!<") - target.write(tagUri) - target.write("> ") + target.append("!<") + target.append(tagUri) + target.append("> ") if anchor != yAnchorNone: - target.write("&") - target.write(anchorName(anchor)) - target.write(' ') + target.append("&") + target.append(anchorName(anchor)) + target.append(' ') except: var e = newException(YamlPresenterOutputError, "") e.parent = getCurrentException() @@ -325,11 +437,12 @@ proc nextItem(c: var Queue, s: var YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} = if c.len > 0: try: result = c.dequeue - except IndexError: assert false + except IndexError: internalError("Unexpected IndexError") else: result = s.next() -proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, +proc doPresent(s: var YamlStream, target: PresenterTarget, + tagLib: TagLibrary, options: PresentationOptions = defaultPresentationOptions) = var indentation = 0 @@ -345,45 +458,45 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, # TODO: tag directives try: case options.outputVersion - of ov1_2: target.write("%YAML 1.2" & newline) - of ov1_1: target.write("%YAML 1.1" & newLine) + of ov1_2: target.append("%YAML 1.2" & newline) + of ov1_1: target.append("%YAML 1.1" & newLine) of ovNone: discard if tagLib.secondaryPrefix != yamlTagRepositoryPrefix: - target.write("%TAG !! " & tagLib.secondaryPrefix & newline) - target.write("--- ") + target.append("%TAG !! " & tagLib.secondaryPrefix & newline) + target.append("--- ") except: var e = newException(YamlPresenterOutputError, "") e.parent = getCurrentException() raise e of yamlScalar: if levels.len == 0: - if options.style != psJson: safeWrite(newline) + if options.style != psJson: target.safeWrite(newline) else: startItem(target, options.style, indentation, levels[levels.high], false, newline) if options.style != psJson: writeTagAndAnchor(target, item.scalarTag, tagLib, item.scalarAnchor) - + if options.style == psJson: let hint = guessType(item.scalarContent) if item.scalarTag in [yTagQuestionMark, yTagBoolean] and hint in {yTypeBoolTrue, yTypeBoolFalse}: - safeWrite(if hint == yTypeBoolTrue: "true" else: "false") + target.safeWrite(if hint == yTypeBoolTrue: "true" else: "false") elif item.scalarTag in [yTagQuestionMark, yTagNull] and hint == yTypeNull: - safeWrite("null") + target.safeWrite("null") elif item.scalarTag in [yTagQuestionMark, yTagInteger, yTagNimInt8, yTagNimInt16, yTagNimInt32, yTagNimInt64, yTagNimUInt8, yTagNimUInt16, yTagNimUInt32, yTagNimUInt64] and hint == yTypeInteger: - safeWrite(item.scalarContent) + target.safeWrite(item.scalarContent) elif item.scalarTag in [yTagQuestionMark, yTagFloat, yTagNimFloat32, yTagNimFloat64] and hint in {yTypeFloatInf, yTypeFloatNaN}: raise newException(YamlPresenterJsonError, "Infinity and not-a-number values cannot be presented as JSON!") elif item.scalarTag in [yTagQuestionMark, yTagFloat] and hint == yTypeFloat: - safeWrite(item.scalarContent) + target.safeWrite(item.scalarContent) else: writeDoubleQuotedJson(item.scalarContent, target) elif options.style == psCanonical: writeDoubleQuoted(item.scalarContent, target, @@ -396,19 +509,19 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, options.indentationStep, target, lines, newline) of sFolded: writeFolded(item.scalarContent, indentation, options.indentationStep, target, words, newline) - of sPlain: safeWrite(item.scalarContent) + of sPlain: target.safeWrite(item.scalarContent) of sDoubleQuoted: writeDoubleQuoted(item.scalarContent, target, indentation + options.indentationStep, newline) of yamlAlias: if options.style == psJson: raise newException(YamlPresenterJsonError, "Alias not allowed in JSON output") - assert levels.len > 0 + yAssert levels.len > 0 startItem(target, options.style, indentation, levels[levels.high], false, newline) try: - target.write('*') - target.write(cast[byte]('a') + cast[byte](item.aliasTarget)) + target.append('*') + target.append(char(byte('a') + byte(item.aliasTarget))) except: var e = newException(YamlPresenterOutputError, "") e.parent = getCurrentException() @@ -419,7 +532,7 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, of psDefault: var length = 0 while true: - assert(not(s.finished())) + yAssert(not s.finished()) let next = s.next() cached.enqueue(next) case next.kind @@ -437,27 +550,27 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, "Cannot have sequence as map key in JSON output!") nextState = dFlowSequenceStart of psMinimal, psCanonical: nextState = dFlowSequenceStart - of psBlockOnly: nextState = dBlockSequenceItem - + of psBlockOnly: nextState = dBlockSequenceItem + if levels.len == 0: case nextState of dBlockSequenceItem: if options.style != psJson: writeTagAndAnchor(target, item.seqTag, tagLib, item.seqAnchor) of dFlowSequenceStart: - safeWrite(newline) + target.safeWrite(newline) if options.style != psJson: writeTagAndAnchor(target, item.seqTag, tagLib, item.seqAnchor) indentation += options.indentationStep - else: assert false + else: internalError("Invalid nextState: " & $nextState) else: startItem(target, options.style, indentation, levels[levels.high], true, newline) if options.style != psJson: writeTagAndAnchor(target, item.seqTag, tagLib, item.seqAnchor) indentation += options.indentationStep - - if nextState == dFlowSequenceStart: safeWrite('[') + + if nextState == dFlowSequenceStart: target.safeWrite('[') if levels.len > 0 and options.style in [psJson, psCanonical] and levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue, dBlockImplicitMapKey, dBlockSequenceItem]: @@ -496,16 +609,16 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, writeTagAndAnchor(target, item.mapTag, tagLib, item.mapAnchor) else: if options.style != psJson: - safeWrite(newline) + target.safeWrite(newline) writeTagAndAnchor(target, item.mapTag, tagLib, item.mapAnchor) indentation += options.indentationStep of dFlowMapStart: - safeWrite(newline) + target.safeWrite(newline) if options.style != psJson: writeTagAndAnchor(target, item.mapTag, tagLib, item.mapAnchor) indentation += options.indentationStep of dBlockInlineMap: discard - else: assert false + else: internalError("Invalid nextState: " & $nextState) else: if nextState in [dBlockMapValue, dBlockImplicitMapKey]: startItem(target, options.style, indentation, @@ -518,27 +631,27 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, if options.style != psJson: writeTagAndAnchor(target, item.mapTag, tagLib, item.mapAnchor) indentation += options.indentationStep - - if nextState == dFlowMapStart: safeWrite('{') + + if nextState == dFlowMapStart: target.safeWrite('{') if levels.len > 0 and options.style in [psJson, psCanonical] and levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue, dBlockImplicitMapKey, dBlockSequenceItem]: indentation += options.indentationStep levels.add(nextState) - + of yamlEndSeq: - assert levels.len > 0 + yAssert levels.len > 0 case levels.pop() of dFlowSequenceItem: case options.style - of psDefault, psMinimal, psBlockOnly: safeWrite(']') + of psDefault, psMinimal, psBlockOnly: target.safeWrite(']') of psJson, psCanonical: indentation -= options.indentationStep try: - target.write(newline) - target.write(repeat(' ', indentation)) - target.write(']') + target.append(newline) + target.append(repeat(' ', indentation)) + target.append(']') except: var e = newException(YamlPresenterOutputError, "") e.parent = getCurrentException() @@ -552,23 +665,23 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue, dBlockImplicitMapKey, dBlockSequenceItem]: indentation -= options.indentationStep - safeWrite(']') + target.safeWrite(']') of dBlockSequenceItem: discard - else: assert false + else: internalError("Invalid popped level") indentation -= options.indentationStep of yamlEndMap: - assert levels.len > 0 + yAssert levels.len > 0 let level = levels.pop() case level of dFlowMapValue: case options.style - of psDefault, psMinimal, psBlockOnly: safeWrite('}') + of psDefault, psMinimal, psBlockOnly: target.safeWrite('}') of psJson, psCanonical: indentation -= options.indentationStep try: - target.write(newline) - target.write(repeat(' ', indentation)) - target.write('}') + target.append(newline) + target.append(repeat(' ', indentation)) + target.append('}') except: var e = newException(YamlPresenterOutputError, "") e.parent = getCurrentException() @@ -582,57 +695,89 @@ proc present*(s: var YamlStream, target: Stream, tagLib: TagLibrary, levels[levels.high] in [dBlockExplicitMapKey, dBlockMapValue, dBlockImplicitMapKey, dBlockSequenceItem]: indentation -= options.indentationStep - safeWrite('}') + target.safeWrite('}') of dBlockMapValue, dBlockInlineMap: discard - else: assert(false) + else: internalError("Invalid level: " & $level) indentation -= options.indentationStep of yamlEndDoc: if finished(s): break if options.style == psJson: raise newException(YamlPresenterJsonError, "Cannot output more than one document in JSON style") - safeWrite("..." & newline) + target.safeWrite("..." & newline) -proc transform*(input: Stream, output: Stream, - options: PresentationOptions = defaultPresentationOptions) = +proc present*(s: var YamlStream, target: Stream, + tagLib: TagLibrary, + options: PresentationOptions = defaultPresentationOptions) + {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} = + ## Convert ``s`` to a YAML character stream and write it to ``target``. + doPresent(s, target, tagLib, options) + +proc present*(s: var YamlStream, tagLib: TagLibrary, + options: PresentationOptions = defaultPresentationOptions): + string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} = + ## Convert ``s`` to a YAML character stream and return it as string. + result = "" + doPresent(s, addr result, tagLib, options) + +proc doTransform(input: Stream | string, output: PresenterTarget, + options: PresentationOptions = defaultPresentationOptions) = var taglib = initExtendedTagLibrary() parser = newYamlParser(tagLib) events = parser.parse(input) try: if options.style == psCanonical: - var specificTagEvents = iterator(): YamlStreamEvent = - for e in events: - var event = e - case event.kind - of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq: - discard - of yamlStartMap: - if event.mapTag in [yTagQuestionMark, yTagExclamationMark]: - event.mapTag = yTagMapping - of yamlStartSeq: - if event.seqTag in [yTagQuestionMark, yTagExclamationMark]: - event.seqTag = yTagSequence - of yamlScalar: - if event.scalarTag == yTagQuestionMark: - case guessType(event.scalarContent) - of yTypeInteger: event.scalarTag = yTagInteger - of yTypeFloat, yTypeFloatInf, yTypeFloatNaN: - event.scalarTag = yTagFloat - of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean - of yTypeNull: event.scalarTag = yTagNull - of yTypeUnknown: event.scalarTag = yTagString - elif event.scalarTag == yTagExclamationMark: - event.scalarTag = yTagString - yield event - var s = initYamlStream(specificTagEvents) - present(s, output, tagLib, options) + var bys: YamlStream = newBufferYamlStream() + for e in events: + var event = e + case event.kind + of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlAlias, yamlEndSeq: + discard + of yamlStartMap: + if event.mapTag in [yTagQuestionMark, yTagExclamationMark]: + event.mapTag = yTagMapping + of yamlStartSeq: + if event.seqTag in [yTagQuestionMark, yTagExclamationMark]: + event.seqTag = yTagSequence + of yamlScalar: + if event.scalarTag == yTagQuestionMark: + case guessType(event.scalarContent) + of yTypeInteger: event.scalarTag = yTagInteger + of yTypeFloat, yTypeFloatInf, yTypeFloatNaN: + event.scalarTag = yTagFloat + of yTypeBoolTrue, yTypeBoolFalse: event.scalarTag = yTagBoolean + of yTypeNull: event.scalarTag = yTagNull + of yTypeUnknown: event.scalarTag = yTagString + elif event.scalarTag == yTagExclamationMark: + event.scalarTag = yTagString + BufferYamlStream(bys).put(e) + present(bys, output, tagLib, options) else: present(events, output, tagLib, options) except YamlStreamError: var e = getCurrentException() while e.parent of YamlStreamError: e = e.parent if e.parent of IOError: raise (ref IOError)(e.parent) elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent) - else: - # never happens - assert(false) + else: internalError("Unexpected exception: " & e.parent.repr) + +proc transform*(input: Stream | string, output: Stream, + options: PresentationOptions = defaultPresentationOptions) + {.raises: [IOError, YamlParserError, YamlPresenterJsonError, + YamlPresenterOutputError].} = + ## Parser ``input`` as YAML character stream and then dump it to ``output`` + ## while resolving non-specific tags to the ones in the YAML core tag + ## library. + doTransform(input, output, options) + +proc transform*(input: Stream | string, + options: PresentationOptions = defaultPresentationOptions): + string {.raises: [IOError, YamlParserError, YamlPresenterJsonError, + YamlPresenterOutputError].} = + ## Parser ``input`` as YAML character stream, resolves non-specific tags to + ## the ones in the YAML core tag library, and then returns a serialized + ## YAML string that represents the stream. + result = "" + doTransform(input, addr result, options) diff --git a/yaml/serialization.nim b/yaml/serialization.nim new file mode 100644 index 0000000..0f384cd --- /dev/null +++ b/yaml/serialization.nim @@ -0,0 +1,1034 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## ========================= +## Module yaml.serialization +## ========================= +## +## This is the most high-level API of NimYAML. It enables you to parse YAML +## character streams directly into native YAML types and vice versa. It builds +## on top of the low-level parser and presenter APIs. +## +## It is possible to define custom construction and serialization procs for any +## type. Please consult the serialization guide on the NimYAML website for more +## information. + +import tables, typetraits, strutils, macros, streams +import parser, taglib, presenter, stream, ../private/internal, hints + +type + SerializationContext* = ref object + ## Context information for the process of serializing YAML from Nim values. + refs*: Table[pointer, AnchorId] + style: AnchorStyle + nextAnchorId*: AnchorId + put*: proc(e: YamlStreamEvent) {.raises: [], closure.} + + ConstructionContext* = ref object + ## Context information for the process of constructing Nim values from YAML. + refs*: Table[AnchorId, pointer] + + YamlConstructionError* = object of YamlLoadingError + ## Exception that may be raised when constructing data objects from a + ## `YamlStream <#YamlStream>`_. The fields ``line``, ``column`` and + ## ``lineContent`` are only available if the costructing proc also does + ## parsing, because otherwise this information is not available to the + ## costruction proc. + +# forward declares + +proc constructChild*[T](s: var YamlStream, c: ConstructionContext, + result: var T) + {.raises: [YamlConstructionError, YamlStreamError].} + ## Constructs an arbitrary Nim value from a part of a YAML stream. + ## The stream will advance until after the finishing token that was used + ## for constructing the value. The ``ConstructionContext`` is needed for + ## potential child objects which may be refs. + +proc constructChild*(s: var YamlStream, c: ConstructionContext, + result: var string) + {.raises: [YamlConstructionError, YamlStreamError].} + ## Constructs a Nim value that is a string from a part of a YAML stream. + ## This specialization takes care of possible nil strings. + +proc constructChild*[T](s: var YamlStream, c: ConstructionContext, + result: var seq[T]) + {.raises: [YamlConstructionError, YamlStreamError].} + ## Constructs a Nim value that is a string from a part of a YAML stream. + ## This specialization takes care of possible nil seqs. + +proc constructChild*[O](s: var YamlStream, c: ConstructionContext, + result: var ref O) + {.raises: [YamlStreamError].} + ## Constructs an arbitrary Nim value from a part of a YAML stream. + ## The stream will advance until after the finishing token that was used + ## for constructing the value. The object may be constructed from an alias + ## node which will be resolved using the ``ConstructionContext``. + +proc representChild*[O](value: ref O, ts: TagStyle, c: SerializationContext) + {.raises: [].} + ## Represents an arbitrary Nim reference value as YAML object. The object + ## may be represented as alias node if it is already present in the + ## ``SerializationContext``. + +proc representChild*(value: string, ts: TagStyle, c: SerializationContext) + {.inline, raises: [].} + ## Represents a Nim string. Supports nil strings. + +proc representChild*[O](value: O, ts: TagStyle, c: SerializationContext) + ## Represents an arbitrary Nim object as YAML object. + +proc newConstructionContext*(): ConstructionContext = + new(result) + result.refs = initTable[AnchorId, pointer]() + +proc newSerializationContext*(s: AnchorStyle, + putImpl: proc(e: YamlStreamEvent) {.raises: [], closure.}): + SerializationContext = + SerializationContext(refs: initTable[pointer, AnchorId](), style: s, + nextAnchorId: 0.AnchorId, put: putImpl) + +template presentTag*(t: typedesc, ts: TagStyle): TagId = + ## Get the TagId that represents the given type in the given style + if ts == tsNone: yTagQuestionMark else: yamlTag(t) + +proc lazyLoadTag(uri: string): TagId {.inline, raises: [].} = + try: result = serializationTagLibrary.tags[uri] + except KeyError: result = serializationTagLibrary.registerUri(uri) + +proc safeTagUri(id: TagId): string {.raises: [].} = + try: + let uri = serializationTagLibrary.uri(id) + if uri.len > 0 and uri[0] == '!': return uri[1..uri.len - 1] + else: return uri + except KeyError: internalError("Unexpected KeyError for TagId " & $id) + +proc constructionError(s: YamlStream, msg: string): ref YamlConstructionError = + result = newException(YamlConstructionError, msg) + if not s.getLastTokenContext(result.line, result.column, result.lineContent): + (result.line, result.column) = (-1, -1) + result.lineContent = "" + +template constructScalarItem*(s: var YamlStream, i: untyped, + t: typedesc, content: untyped) = + ## Helper template for implementing ``constructObject`` for types that + ## are constructed from a scalar. ``i`` is the identifier that holds + ## the scalar as ``YamlStreamEvent`` in the content. Exceptions raised in + ## the content will be automatically catched and wrapped in + ## ``YamlConstructionError``, which will then be raised. + bind constructionError + let i = s.next() + if i.kind != yamlScalar: + raise constructionError(s, "Expected scalar") + try: content + except YamlConstructionError: raise + except Exception: + var e = constructionError(s, + "Cannot construct to " & name(t) & ": " & item.scalarContent) + e.parent = getCurrentException() + raise e + +proc yamlTag*(T: typedesc[string]): TagId {.inline, noSideEffect, raises: [].} = + yTagString + +proc constructObject*(s: var YamlStream, c: ConstructionContext, + result: var string) + {.raises: [YamlConstructionError, YamlStreamError].} = + ## costructs a string from a YAML scalar + constructScalarItem(s, item, string): + result = item.scalarContent + +proc representObject*(value: string, ts: TagStyle, + c: SerializationContext, tag: TagId) {.raises: [].} = + ## represents a string as YAML scalar + c.put(scalarEvent(value, tag, yAnchorNone)) + +proc parseHex[T: int8|int16|int32|int64|uint8|uint16|uint32|uint64]( + s: YamlStream, val: string): T = + result = 0 + for i in 2.. 0: + representChild(field, if ts == tsAll: tsAll else: tsRootOnly, c) + inc(count) + if count == 1: c.put(scalarEvent("~", yTagNull)) + else: + representObject(value, ts, c, + if ts == tsNone: yTagQuestionMark else: yamlTag(O)) + +proc construct*[T](s: var YamlStream, target: var T) + {.raises: [YamlStreamError].} = + ## Constructs a Nim value from a YAML stream. + var context = newConstructionContext() + try: + var e = s.next() + yAssert(e.kind == yamlStartDoc) + + constructChild(s, context, target) + e = s.next() + yAssert(e.kind == yamlEndDoc) + except YamlConstructionError: + raise (ref YamlConstructionError)(getCurrentException()) + except YamlStreamError: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur.parent + raise e + except Exception: + # may occur while calling s() + var ex = newException(YamlStreamError, "") + ex.parent = getCurrentException() + raise ex + +proc load*[K](input: Stream | string, target: var K) + {.raises: [YamlConstructionError, IOError, YamlParserError].} = + ## Loads a Nim value from a YAML character stream. + var + parser = newYamlParser(serializationTagLibrary) + events = parser.parse(input) + try: construct(events, target) + except YamlStreamError: + let e = (ref YamlStreamError)(getCurrentException()) + if e.parent of IOError: raise (ref IOError)(e.parent) + elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent) + else: internalError("Unexpected exception: " & e.parent.repr) + +proc loadMultiDoc*[K](input: Stream | string, target: var seq[K]) = + if target.isNil: + target = newSeq[K]() + var + parser = newYamlParser(serializationTagLibrary) + events = parser.parse(input) + try: + while not events.finished(): + var item: K + construct(events, item) + target.add(item) + except YamlConstructionError: + var e = (ref YamlConstructionError)(getCurrentException()) + discard events.getLastTokenContext(e.line, e.column, e.lineContent) + raise e + except YamlStreamError: + let e = (ref YamlStreamError)(getCurrentException()) + if e.parent of IOError: raise (ref IOError)(e.parent) + elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent) + else: internalError("Unexpected exception: " & e.parent.repr) + +proc setAnchor(a: var AnchorId, q: var Table[pointer, AnchorId]) + {.inline.} = + if a != yAnchorNone: a = q.getOrDefault(cast[pointer](a)) + +proc represent*[T](value: T, ts: TagStyle = tsRootOnly, + a: AnchorStyle = asTidy): YamlStream = + ## Represents a Nim value as ``YamlStream`` + var bys = newBufferYamlStream() + var context = newSerializationContext(a, proc(e: YamlStreamEvent) = + bys.put(e) + ) + bys.put(startDocEvent()) + representChild(value, ts, context) + bys.put(endDocEvent()) + if a == asTidy: + for item in bys.mitems(): + case item.kind + of yamlStartMap: item.mapAnchor.setAnchor(context.refs) + of yamlStartSeq: item.seqAnchor.setAnchor(context.refs) + of yamlScalar: item.scalarAnchor.setAnchor(context.refs) + else: discard + result = bys + +proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly, + anchorStyle: AnchorStyle = asTidy, + options: PresentationOptions = defaultPresentationOptions) + {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} = + ## Dump a Nim value as YAML character stream. + var events = represent(value, + if options.style == psCanonical: tsAll else: tagStyle, + if options.style == psJson: asNone else: anchorStyle) + try: present(events, target, serializationTagLibrary, options) + except YamlStreamError: + internalError("Unexpected exception: " & getCurrentException().repr) + +proc dump*[K](value: K, tagStyle: TagStyle = tsRootOnly, + anchorStyle: AnchorStyle = asTidy, + options: PresentationOptions = defaultPresentationOptions): + string = + ## Dump a Nim value as YAML into a string + var events = represent(value, + if options.style == psCanonical: tsAll else: tagStyle, + if options.style == psJson: asNone else: anchorStyle) + try: result = present(events, serializationTagLibrary, options) + except YamlStreamError: + internalError("Unexpected exception: " & getCurrentException().repr) diff --git a/yaml/stream.nim b/yaml/stream.nim new file mode 100644 index 0000000..9cc2e0b --- /dev/null +++ b/yaml/stream.nim @@ -0,0 +1,305 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## ================== +## Module yaml.stream +## ================== +## +## The stream API provides the basic data structure on which all low-level APIs +## operate. It is not named ``streams`` to not confuse it with the modle in the +## stdlib with that name. + +import hashes +import ../private/internal, taglib + +type + AnchorId* = distinct int ## \ + ## An ``AnchorId`` identifies an anchor in the current document. It + ## becomes invalid as soon as the current document scope is invalidated + ## (for example, because the parser yielded a ``yamlEndDocument`` + ## event). ``AnchorId`` s exists because of efficiency, much like + ## ``TagId`` s. The actual anchor name is a presentation detail and + ## cannot be queried by the user. + + YamlStreamEventKind* = enum + ## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds + ## are discussed in `YamlStreamEvent <#YamlStreamEvent>`_. + yamlStartDoc, yamlEndDoc, yamlStartMap, yamlEndMap, + yamlStartSeq, yamlEndSeq, yamlScalar, yamlAlias + + YamlStreamEvent* = object + ## An element from a `YamlStream <#YamlStream>`_. Events that start an + ## object (``yamlStartMap``, ``yamlStartSeq``, ``yamlScalar``) have + ## an optional anchor and a tag associated with them. The anchor will be + ## set to ``yAnchorNone`` if it doesn't exist. + ## + ## A non-existing tag in the YAML character stream will be resolved to + ## the non-specific tags ``?`` or ``!`` according to the YAML + ## specification. These are by convention mapped to the ``TagId`` s + ## ``yTagQuestionMark`` and ``yTagExclamationMark`` respectively. + ## Mapping is done by a `TagLibrary <#TagLibrary>`_. + case kind*: YamlStreamEventKind + of yamlStartMap: + mapAnchor* : AnchorId + mapTag* : TagId + of yamlStartSeq: + seqAnchor* : AnchorId + seqTag* : TagId + of yamlScalar: + scalarAnchor* : AnchorId + scalarTag* : TagId + scalarContent*: string # may not be nil (but empty) + of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard + of yamlAlias: + aliasTarget* : AnchorId + + YamlStream* = ref object of RootObj ## \ + ## A ``YamlStream`` is an iterator-like object that yields a + ## well-formed stream of ``YamlStreamEvents``. Well-formed means that + ## every ``yamlStartMap`` is terminated by a ``yamlEndMap``, every + ## ``yamlStartSeq`` is terminated by a ``yamlEndSeq`` and every + ## ``yamlStartDoc`` is terminated by a ``yamlEndDoc``. Moreover, every + ## emitted mapping has an even number of children. + ## + ## The creator of a ``YamlStream`` is responsible for it being + ## well-formed. A user of the stream may assume that it is well-formed + ## and is not required to check for it. The procs in this module will + ## always yield a well-formed ``YamlStream`` and expect it to be + ## well-formed if they take it as input parameter. + nextImpl*: proc(s: YamlStream, e: var YamlStreamEvent): bool + lastTokenContextImpl*: + proc(s: YamlStream, line, column: var int, + lineContent: var string): bool {.raises: [].} + isFinished*: bool + peeked: bool + cached: YamlStreamEvent + + YamlStreamError* = object of Exception + ## Exception that may be raised by a ``YamlStream`` when the underlying + ## backend raises an exception. The error that has occurred is + ## available from ``parent``. + +const + yAnchorNone*: AnchorId = (-1).AnchorId ## \ + ## yielded when no anchor was defined for a YAML node + +proc `==`*(left, right: AnchorId): bool {.borrow.} +proc `$`*(id: AnchorId): string {.borrow.} +proc hash*(id: AnchorId): Hash {.borrow.} + +proc noLastContext(s: YamlStream, line, column: var int, + lineContent: var string): bool {.raises: [].} = + (line, column, lineContent) = (-1, -1, "") + result = false + +proc basicInit*(s: YamlStream, lastTokenContextImpl: + proc(s: YamlStream, line, column: var int, lineContent: var string): bool + {.raises: [].} = noLastContext) + {.raises: [].} = + ## initialize basic values of the YamlStream. Call this in your constructor + ## if you subclass YamlStream. + s.peeked = false + s.isFinished = false + s.lastTokenContextImpl = lastTokenContextImpl + +when not defined(JS): + type IteratorYamlStream = ref object of YamlStream + backend: iterator(): YamlStreamEvent + + proc initYamlStream*(backend: iterator(): YamlStreamEvent): YamlStream + {.raises: [].} = + ## Creates a new ``YamlStream`` that uses the given iterator as backend. + result = new(IteratorYamlStream) + result.basicInit() + IteratorYamlStream(result).backend = backend + result.nextImpl = proc(s: YamlStream, e: var YamlStreamEvent): bool = + e = IteratorYamlStream(s).backend() + if finished(IteratorYamlStream(s).backend): + s.isFinished = true + result = false + else: result = true + +type + BufferYamlStream* = ref object of YamlStream + pos: int + buf: seq[YamlStreamEvent] not nil + +proc newBufferYamlStream*(): BufferYamlStream not nil = + result = cast[BufferYamlStream not nil](new(BufferYamlStream)) + result.basicInit() + result.buf = @[] + result.pos = 0 + result.nextImpl = proc(s: YamlStream, e: var YamlStreamEvent): bool = + let bys = BufferYamlStream(s) + if bys.pos == bys.buf.len: + result = false + s.isFinished = true + else: + e = bys.buf[bys.pos] + inc(bys.pos) + result = true + +proc put*(bys: BufferYamlStream, e: YamlStreamEvent) {.raises: [].} = + bys.buf.add(e) + +proc next*(s: YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} = + ## Get the next item of the stream. Requires ``finished(s) == true``. + ## If the backend yields an exception, that exception will be encapsulated + ## into a ``YamlStreamError``, which will be raised. + if s.peeked: + s.peeked = false + shallowCopy(result, s.cached) + return + else: + yAssert(not s.isFinished) + try: + while true: + if s.nextImpl(s, result): break + yAssert(not s.isFinished) + except YamlStreamError: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur.parent + raise e + except Exception: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur + raise e + +proc peek*(s: YamlStream): YamlStreamEvent {.raises: [YamlStreamError].} = + ## Get the next item of the stream without advancing the stream. + ## Requires ``finished(s) == true``. Handles exceptions of the backend like + ## ``next()``. + if not s.peeked: + shallowCopy(s.cached, s.next()) + s.peeked = true + shallowCopy(result, s.cached) + +proc `peek=`*(s: YamlStream, value: YamlStreamEvent) {.raises: [].} = + ## Set the next item of the stream. Will replace a previously peeked item, + ## if one exists. + s.cached = value + s.peeked = true + +proc finished*(s: YamlStream): bool {.raises: [YamlStreamError].} = + ## ``true`` if no more items are available in the stream. Handles exceptions + ## of the backend like ``next()``. + if s.peeked: result = false + else: + try: + while true: + if s.isFinished: return true + if s.nextImpl(s, s.cached): + s.peeked = true + return false + except YamlStreamError: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur.parent + raise e + except Exception: + let cur = getCurrentException() + var e = newException(YamlStreamError, cur.msg) + e.parent = cur + raise e + +proc getLastTokenContext*(s: YamlStream, line, column: var int, + lineContent: var string): bool = + ## ``true`` if source context information is available about the last returned + ## token. If ``true``, line, column and lineContent are set to position and + ## line content where the last token has been read from. + result = s.lastTokenContextImpl(s, line, column, lineContent) + +iterator items*(s: YamlStream): YamlStreamEvent + {.raises: [YamlStreamError].} = + ## Iterate over all items of the stream. You may not use ``peek()`` on the + ## stream while iterating. + while not s.finished(): yield s.next() + +iterator mitems*(bys: BufferYamlStream): var YamlStreamEvent {.raises: [].} = + ## Iterate over all items of the stream. You may not use ``peek()`` on the + ## stream while iterating. + for e in bys.buf.mitems(): yield e + +proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool {.raises: [].} = + ## compares all existing fields of the given items + if left.kind != right.kind: return false + case left.kind + of yamlStartDoc, yamlEndDoc, yamlEndMap, yamlEndSeq: result = true + of yamlStartMap: + result = left.mapAnchor == right.mapAnchor and left.mapTag == right.mapTag + of yamlStartSeq: + result = left.seqAnchor == right.seqAnchor and left.seqTag == right.seqTag + of yamlScalar: + result = left.scalarAnchor == right.scalarAnchor and + left.scalarTag == right.scalarTag and + left.scalarContent == right.scalarContent + of yamlAlias: result = left.aliasTarget == right.aliasTarget + +proc `$`*(event: YamlStreamEvent): string {.raises: [].} = + ## outputs a human-readable string describing the given event + result = $event.kind & '(' + case event.kind + of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard + of yamlStartMap: + result &= "tag=" & $event.mapTag + if event.mapAnchor != yAnchorNone: result &= ", anchor=" & $event.mapAnchor + of yamlStartSeq: + result &= "tag=" & $event.seqTag + if event.seqAnchor != yAnchorNone: result &= ", anchor=" & $event.seqAnchor + of yamlScalar: + result &= "tag=" & $event.scalarTag + if event.scalarAnchor != yAnchorNone: + result &= ", anchor=" & $event.scalarAnchor + result &= ", content=\"" & event.scalarContent & '\"' + of yamlAlias: + result &= "aliasTarget=" & $event.aliasTarget + result &= ")" + +proc tag*(event: YamlStreamEvent): TagId {.raises: [FieldError].} = + ## returns the tag of the given event + case event.kind + of yamlStartMap: result = event.mapTag + of yamlStartSeq: result = event.seqTag + of yamlScalar: result = event.scalarTag + else: raise newException(FieldError, "Event " & $event.kind & " has no tag") + +proc startDocEvent*(): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the start of a YAML document + result = YamlStreamEvent(kind: yamlStartDoc) + +proc endDocEvent*(): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the end of a YAML document + result = YamlStreamEvent(kind: yamlEndDoc) + +proc startMapEvent*(tag: TagId = yTagQuestionMark, + anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the start of a YAML mapping + result = YamlStreamEvent(kind: yamlStartMap, mapTag: tag, mapAnchor: anchor) + +proc endMapEvent*(): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the end of a YAML mapping + result = YamlStreamEvent(kind: yamlEndMap) + +proc startSeqEvent*(tag: TagId = yTagQuestionMark, + anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the beginning of a YAML sequence + result = YamlStreamEvent(kind: yamlStartSeq, seqTag: tag, seqAnchor: anchor) + +proc endSeqEvent*(): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that marks the end of a YAML sequence + result = YamlStreamEvent(kind: yamlEndSeq) + +proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, + anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that represents a YAML scalar + result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag, + scalarAnchor: anchor, scalarContent: content) + +proc aliasEvent*(anchor: AnchorId): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that represents a YAML alias + result = YamlStreamEvent(kind: yamlAlias, aliasTarget: anchor) diff --git a/yaml/taglib.nim b/yaml/taglib.nim new file mode 100644 index 0000000..5a2ddcc --- /dev/null +++ b/yaml/taglib.nim @@ -0,0 +1,309 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +## ================== +## Module yaml.taglib +## ================== +## +## The taglib API enables you to query real names of tags emitted by the parser +## and create own tags. It also enables you to define tags for types used with +## the serialization API. + +import tables, macros, hashes + +type + TagId* = distinct int ## \ + ## A ``TagId`` identifies a tag URI, like for example + ## ``"tag:yaml.org,2002:str"``. The URI corresponding to a ``TagId`` can + ## be queried from the `TagLibrary <#TagLibrary>`_ which was + ## used to create this ``TagId``; e.g. when you parse a YAML character + ## stream, the ``TagLibrary`` of the parser is the one which generates + ## the resulting ``TagId`` s. + ## + ## URI strings are mapped to ``TagId`` s for efficiency reasons (you + ## do not need to compare strings every time) and to be able to + ## discover unknown tag URIs early in the parsing process. + + TagLibrary* = ref object + ## A ``TagLibrary`` maps tag URIs to ``TagId`` s. + ## + ## When `YamlParser <#YamlParser>`_ encounters tags not existing in the + ## tag library, it will use + ## `registerUri <#registerUri,TagLibrary,string>`_ to add + ## the tag to the library. + ## + ## You can base your tag library on common tag libraries by initializing + ## them with `initFailsafeTagLibrary <#initFailsafeTagLibrary>`_, + ## `initCoreTagLibrary <#initCoreTagLibrary>`_ or + ## `initExtendedTagLibrary <#initExtendedTagLibrary>`_. + tags*: Table[string, TagId] + nextCustomTagId*: TagId + secondaryPrefix*: string + +const + # failsafe schema + + yTagExclamationMark*: TagId = 0.TagId ## ``!`` non-specific tag + yTagQuestionMark* : TagId = 1.TagId ## ``?`` non-specific tag + yTagString* : TagId = 2.TagId ## \ + ## `!!str `_ tag + yTagSequence* : TagId = 3.TagId ## \ + ## `!!seq `_ tag + yTagMapping* : TagId = 4.TagId ## \ + ## `!!map `_ tag + + # json & core schema + + yTagNull* : TagId = 5.TagId ## \ + ## `!!null `_ tag + yTagBoolean* : TagId = 6.TagId ## \ + ## `!!bool `_ tag + yTagInteger* : TagId = 7.TagId ## \ + ## `!!int `_ tag + yTagFloat* : TagId = 8.TagId ## \ + ## `!!float `_ tag + + # other language-independent YAML types (from http://yaml.org/type/ ) + + yTagOrderedMap* : TagId = 9.TagId ## \ + ## `!!omap `_ tag + yTagPairs* : TagId = 10.TagId ## \ + ## `!!pairs `_ tag + yTagSet* : TagId = 11.TagId ## \ + ## `!!set `_ tag + yTagBinary* : TagId = 12.TagId ## \ + ## `!!binary `_ tag + yTagMerge* : TagId = 13.TagId ## \ + ## `!!merge `_ tag + yTagTimestamp* : TagId = 14.TagId ## \ + ## `!!timestamp `_ tag + yTagValue* : TagId = 15.TagId ## \ + ## `!!value `_ tag + yTagYaml* : TagId = 16.TagId ## \ + ## `!!yaml `_ tag + + yTagNimField* : TagId = 100.TagId ## \ + ## This tag is used in serialization for the name of a field of an + ## object. It may contain any string scalar that is a valid Nim symbol. + + yTagNimNilString* : TagId = 101.TagId ## for strings that are nil + yTagNimNilSeq* : TagId = 102.TagId ## \ + ## for seqs that are nil. This tag is used regardless of the seq's generic + ## type parameter. + + yFirstCustomTagId* : TagId = 1000.TagId ## \ + ## The first ``TagId`` which should be assigned to an URI that does not + ## exist in the ``YamlTagLibrary`` which is used for parsing. + + yamlTagRepositoryPrefix* = "tag:yaml.org,2002:" + +proc `==`*(left, right: TagId): bool {.borrow.} +proc hash*(id: TagId): Hash {.borrow.} + +proc `$`*(id: TagId): string {.raises: [].} = + case id + of yTagQuestionMark: "?" + of yTagExclamationMark: "!" + of yTagString: "!!str" + of yTagSequence: "!!seq" + of yTagMapping: "!!map" + of yTagNull: "!!null" + of yTagBoolean: "!!bool" + of yTagInteger: "!!int" + of yTagFloat: "!!float" + of yTagOrderedMap: "!!omap" + of yTagPairs: "!!pairs" + of yTagSet: "!!set" + of yTagBinary: "!!binary" + of yTagMerge: "!!merge" + of yTagTimestamp: "!!timestamp" + of yTagValue: "!!value" + of yTagYaml: "!!yaml" + of yTagNimField: "!nim:field" + else: "<" & $int(id) & ">" + +proc initTagLibrary*(): TagLibrary {.raises: [].} = + ## initializes the ``tags`` table and sets ``nextCustomTagId`` to + ## ``yFirstCustomTagId``. + new(result) + result.tags = initTable[string, TagId]() + result.secondaryPrefix = yamlTagRepositoryPrefix + result.nextCustomTagId = yFirstCustomTagId + +proc registerUri*(tagLib: TagLibrary, uri: string): TagId {.raises: [].} = + ## registers a custom tag URI with a ``TagLibrary``. The URI will get + ## the ``TagId`` ``nextCustomTagId``, which will be incremented. + tagLib.tags[uri] = tagLib.nextCustomTagId + result = tagLib.nextCustomTagId + tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1) + +proc uri*(tagLib: TagLibrary, id: TagId): string {.raises: [KeyError].} = + ## retrieve the URI a ``TagId`` maps to. + for iUri, iId in tagLib.tags.pairs: + if iId == id: return iUri + raise newException(KeyError, "Unknown tag id: " & $id) + +proc initFailsafeTagLibrary*(): TagLibrary {.raises: [].} = + ## Contains only: + ## - ``!`` + ## - ``?`` + ## - ``!!str`` + ## - ``!!map`` + ## - ``!!seq`` + result = initTagLibrary() + result.tags["!"] = yTagExclamationMark + result.tags["?"] = yTagQuestionMark + result.tags["tag:yaml.org,2002:str"] = yTagString + result.tags["tag:yaml.org,2002:seq"] = yTagSequence + result.tags["tag:yaml.org,2002:map"] = yTagMapping + +proc initCoreTagLibrary*(): TagLibrary {.raises: [].} = + ## Contains everything in ``initFailsafeTagLibrary`` plus: + ## - ``!!null`` + ## - ``!!bool`` + ## - ``!!int`` + ## - ``!!float`` + result = initFailsafeTagLibrary() + result.tags["tag:yaml.org,2002:null"] = yTagNull + result.tags["tag:yaml.org,2002:bool"] = yTagBoolean + result.tags["tag:yaml.org,2002:int"] = yTagInteger + result.tags["tag:yaml.org,2002:float"] = yTagFloat + +proc initExtendedTagLibrary*(): TagLibrary {.raises: [].} = + ## Contains everything from ``initCoreTagLibrary`` plus: + ## - ``!!omap`` + ## - ``!!pairs`` + ## - ``!!set`` + ## - ``!!binary`` + ## - ``!!merge`` + ## - ``!!timestamp`` + ## - ``!!value`` + ## - ``!!yaml`` + result = initCoreTagLibrary() + result.tags["tag:yaml.org,2002:omap"] = yTagOrderedMap + result.tags["tag:yaml.org,2002:pairs"] = yTagPairs + result.tags["tag:yaml.org,2002:binary"] = yTagBinary + result.tags["tag:yaml.org,2002:merge"] = yTagMerge + result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp + result.tags["tag:yaml.org,2002:value"] = yTagValue + result.tags["tag:yaml.org,2002:yaml"] = yTagYaml + + +proc initSerializationTagLibrary*(): TagLibrary = + result = initTagLibrary() + result.tags["!"] = yTagExclamationMark + result.tags["?"] = yTagQuestionMark + result.tags["tag:yaml.org,2002:str"] = yTagString + result.tags["tag:yaml.org,2002:null"] = yTagNull + result.tags["tag:yaml.org,2002:bool"] = yTagBoolean + result.tags["tag:yaml.org,2002:float"] = yTagFloat + result.tags["tag:yaml.org,2002:timestamp"] = yTagTimestamp + result.tags["tag:yaml.org,2002:value"] = yTagValue + result.tags["tag:yaml.org,2002:binary"] = yTagBinary + result.tags["!nim:field"] = yTagNimField + result.tags["!nim:nil:string"] = yTagNimNilString + result.tags["!nim:nil:seq"] = yTagNimNilSeq + +var + serializationTagLibrary* = initSerializationTagLibrary() ## \ + ## contains all local tags that are used for type serialization. Does + ## not contain any of the specific default tags for sequences or maps, + ## as those are not suited for Nim's static type system. + ## + ## Should not be modified manually. Will be extended by + ## `serializable <#serializable,stmt,stmt>`_. + +var + nextStaticTagId {.compileTime.} = 100.TagId ## \ + ## used for generating unique TagIds with ``setTagUri``. + registeredUris {.compileTime.} = newSeq[string]() ## \ + ## Since Table doesn't really work at compile time, we also store + ## registered URIs here to be able to generate a static compiler error + ## when the user tries to register an URI more than once. + +template setTagUri*(t: typedesc, uri: string): typed = + ## Associate the given uri with a certain type. This uri is used as YAML tag + ## when loading and dumping values of this type. + when uri in registeredUris: + {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .} + const id {.genSym.} = nextStaticTagId + static: + registeredUris.add(uri) + nextStaticTagId = TagId(int(nextStaticTagId) + 1) + when nextStaticTagId == yFirstCustomTagId: + {.fatal: "Too many tags!".} + serializationTagLibrary.tags[uri] = id + proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = id + ## autogenerated + +template setTagUri*(t: typedesc, uri: string, idName: untyped): typed = + ## Like `setTagUri <#setTagUri.t,typedesc,string>`_, but lets + ## you choose a symbol for the `TagId <#TagId>`_ of the uri. This is only + ## necessary if you want to implement serialization / construction yourself. + when uri in registeredUris: + {. fatal: "[NimYAML] URI \"" & uri & "\" registered twice!" .} + const idName* = nextStaticTagId + static: + registeredUris.add(uri) + nextStaticTagId = TagId(int(nextStaticTagId) + 1) + serializationTagLibrary.tags[uri] = idName + proc yamlTag*(T: typedesc[t]): TagId {.inline, raises: [].} = idName + ## autogenerated + +proc canBeImplicit(t: typedesc): bool {.compileTime.} = + let tDesc = getType(t) + if tDesc.kind != nnkObjectTy: return false + if tDesc[2].len != 1: return false + if tDesc[2][0].kind != nnkRecCase: return false + var foundEmptyBranch = false + for i in 1.. tDesc[2][0].len - 1: + case tDesc[2][0][i][1].len # branch contents + of 0: + if foundEmptyBranch: return false + else: foundEmptyBranch = true + of 1: discard + else: return false + return true + +template markAsImplicit*(t: typedesc): typed = + ## Mark a variant object type as implicit. This requires the type to consist + ## of nothing but a case expression and each branch of the case expression + ## containing exactly one field - with the exception that one branch may + ## contain zero fields. + when canBeImplicit(t): + # this will be checked by means of compiles(implicitVariantObject(...)) + proc implicitVariantObject*(unused: t) = discard + else: + {. fatal: "This type cannot be marked as implicit" .} + +static: + # standard YAML tags used by serialization + registeredUris.add("!") + registeredUris.add("?") + registeredUris.add("tag:yaml.org,2002:str") + registeredUris.add("tag:yaml.org,2002:null") + registeredUris.add("tag:yaml.org,2002:bool") + registeredUris.add("tag:yaml.org,2002:float") + registeredUris.add("tag:yaml.org,2002:timestamp") + registeredUris.add("tag:yaml.org,2002:value") + registeredUris.add("tag:yaml.org,2002:binary") + # special tags used by serialization + registeredUris.add("!nim:field") + registeredUris.add("!nim:nil:string") + registeredUris.add("!nim:nil:seq") + +# tags for Nim's standard types +setTagUri(char, "!nim:system:char", yTagNimChar) +setTagUri(int8, "!nim:system:int8", yTagNimInt8) +setTagUri(int16, "!nim:system:int16", yTagNimInt16) +setTagUri(int32, "!nim:system:int32", yTagNimInt32) +setTagUri(int64, "!nim:system:int64", yTagNimInt64) +setTagUri(uint8, "!nim:system:uint8", yTagNimUInt8) +setTagUri(uint16, "!nim:system:uint16", yTagNimUInt16) +setTagUri(uint32, "!nim:system:uint32", yTagNimUInt32) +setTagUri(uint64, "!nim:system:uint64", yTagNimUInt64) +setTagUri(float32, "!nim:system:float32", yTagNimFloat32) +setTagUri(float64, "!nim:system:float64", yTagNimFloat64) \ No newline at end of file diff --git a/private/json.nim b/yaml/tojson.nim similarity index 65% rename from private/json.nim rename to yaml/tojson.nim index f1eb2ae..9c6350f 100644 --- a/private/json.nim +++ b/yaml/tojson.nim @@ -1,9 +1,19 @@ # NimYAML - YAML implementation in Nim -# (c) Copyright 2015 Felix Krause +# (c) Copyright 2016 Felix Krause # # See the file "copying.txt", included in this # distribution, for details about the copyright. +## ================== +## Module yaml.tojson +## ================== +## +## The tojson API enables you to parser a YAML character stream into the JSON +## structures provided by Nim's stdlib. + +import json, streams, strutils, tables +import taglib, hints, serialization, stream, ../private/internal, parser + type Level = tuple[node: JsonNode, key: string] proc initLevel(node: JsonNode): Level {.raises: [].} = @@ -13,7 +23,7 @@ proc jsonFromScalar(content: string, tag: TagId): JsonNode {.raises: [YamlConstructionError].}= new(result) var mappedType: TypeHint - + case tag of yTagQuestionMark: mappedType = guessType(content) of yTagExclamationMark, yTagString: mappedType = yTypeUnknown @@ -35,7 +45,7 @@ proc jsonFromScalar(content: string, tag: TagId): JsonNode raise newException(YamlConstructionError, "Invalid float value: " & content) else: mappedType = yTypeUnknown - + try: case mappedType of yTypeInteger: @@ -66,9 +76,22 @@ proc jsonFromScalar(content: string, tag: TagId): JsonNode e.parent = getCurrentException() raise e -proc constructJson*(s: var YamlStream): seq[JsonNode] = +proc constructJson*(s: var YamlStream): seq[JsonNode] + {.raises: [YamlConstructionError, YamlStreamError].} = + ## Construct an in-memory JSON tree from a YAML event stream. The stream may + ## not contain any tags apart from those in ``coreTagLibrary``. Anchors and + ## aliases will be resolved. Maps in the input must not contain + ## non-scalars as keys. Each element of the result represents one document + ## in the YAML stream. + ## + ## **Warning:** The special float values ``[+-]Inf`` and ``NaN`` will be + ## parsed into Nim's JSON structure without error. However, they cannot be + ## rendered to a JSON character stream, because these values are not part + ## of the JSON specification. Nim's JSON implementation currently does not + ## check for these values and will output invalid JSON when rendering one + ## of these values into a JSON character stream. newSeq(result, 0) - + var levels = newSeq[Level]() anchors = initTable[AnchorId, JsonNode]() @@ -95,7 +118,7 @@ proc constructJson*(s: var YamlStream): seq[JsonNode] = levels.add((node: jsonFromScalar(event.scalarContent, event.scalarTag), key: nil)) continue - + case levels[levels.high].node.kind of JArray: let jsonScalar = jsonFromScalar(event.scalarContent, @@ -117,7 +140,8 @@ proc constructJson*(s: var YamlStream): seq[JsonNode] = levels[levels.high].key = nil if event.scalarAnchor != yAnchorNone: anchors[event.scalarAnchor] = jsonScalar - else: discard # will never happen + else: + internalError("Unexpected node kind: " & $levels[levels.high].node.kind) of yamlEndSeq, yamlEndMap: if levels.len > 1: let level = levels.pop() @@ -130,56 +154,57 @@ proc constructJson*(s: var YamlStream): seq[JsonNode] = else: levels[levels.high].node[levels[levels.high].key] = level.node levels[levels.high].key = nil - else: discard # will never happen + else: + internalError("Unexpected node kind: " & + $levels[levels.high].node.kind) else: discard # wait for yamlEndDocument of yamlAlias: # we can savely assume that the alias exists in anchors # (else the parser would have already thrown an exception) case levels[levels.high].node.kind of JArray: - try: - levels[levels.high].node.elems.add(anchors[event.aliasTarget]) - except KeyError: - # we can safely assume that this doesn't happen. It would - # have resulted in a parser error earlier. - assert(false) + levels[levels.high].node.elems.add( + anchors.getOrDefault(event.aliasTarget)) of JObject: if isNil(levels[levels.high].key): raise newException(YamlConstructionError, "cannot use alias node as key in JSON") else: - try: - levels[levels.high].node.fields.add( - levels[levels.high].key, anchors[event.aliasTarget]) - except KeyError: - # we can safely assume that this doesn't happen. It would - # have resulted in a parser error earlier. - assert(false) + levels[levels.high].node.fields.add( + levels[levels.high].key, anchors.getOrDefault(event.aliasTarget)) levels[levels.high].key = nil - else: discard # will never happen + else: + internalError("Unexpected node kind: " & $levels[levels.high].node.kind) -proc loadToJson*(s: Stream): seq[JsonNode] = +proc loadToJson*(s: Stream): seq[JsonNode] + {.raises: [YamlParserError, YamlConstructionError, IOError].} = + ## Uses `YamlParser <#YamlParser>`_ and + ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree + ## from a YAML character stream. var parser = newYamlParser(initCoreTagLibrary()) events = parser.parse(s) try: return constructJson(events) - except YamlConstructionError: - var e = cast[ref YamlConstructionError](getCurrentException()) - e.line = parser.getLineNumber() - e.column = parser.getColNumber() - e.lineContent = parser.getLineContent() - raise e except YamlStreamError: let e = getCurrentException() if e.parent of IOError: - raise cast[ref IOError](e.parent) + raise (ref IOError)(e.parent) elif e.parent of YamlParserError: - raise cast[ref YamlParserError](e.parent) - else: - # can never happen - assert(false) - except AssertionError: raise - except Exception: - # compiler bug: https://github.com/nim-lang/Nim/issues/3772 - assert(false) \ No newline at end of file + raise (ref YamlParserError)(e.parent) + else: internalError("Unexpected exception: " & e.parent.repr) + +proc loadToJson*(str: string): seq[JsonNode] + {.raises: [YamlParserError, YamlConstructionError].} = + ## Uses `YamlParser <#YamlParser>`_ and + ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree + ## from a YAML character stream. + var + parser = newYamlParser(initCoreTagLibrary()) + events = parser.parse(str) + try: return constructJson(events) + except YamlStreamError: + let e = getCurrentException() + if e.parent of YamlParserError: + raise (ref YamlParserError)(e.parent) + else: internalError("Unexpected exception: " & e.parent.repr) \ No newline at end of file