diff --git a/src/private/json.nim b/src/private/json.nim index 8940f84..fd58ee8 100644 --- a/src/private/json.nim +++ b/src/private/json.nim @@ -3,9 +3,28 @@ type proc initLevel(node: JsonNode): Level = (node: node, key: cast[string](nil)) -proc jsonFromScalar(content: string, typeHint: YamlTypeHint): JsonNode = +proc jsonFromScalar(content: string, tag: TagId, + typeHint: YamlTypeHint): JsonNode = new(result) - case typeHint + var mappedType: YamlTypeHint + + case tag + of tagQuestionMark: + mappedType = typeHint + of tagExclamationMark, tagString: + mappedType = yTypeString + of tagBoolean: + mappedType = yTypeBoolean + of tagInteger: + mappedType = yTypeInteger + of tagNull: + mappedType = yTypeNull + of tagFloat: + mappedType = yTypeFloat + else: + mappedType = yTypeUnknown + + case mappedType of yTypeInteger: result.kind = JInt result.num = parseBiggestInt(content) @@ -28,15 +47,10 @@ proc parseToJson*(s: Stream): seq[JsonNode] = newSeq(result, 0) var - levels = newSeq[Level]() - parser = newParser() - tagStr = parser.registerUri("tag:yaml.org,2002:str") - tagBool = parser.registerUri("tag:yaml.org,2002:bool") - tagNull = parser.registerUri("tag:yaml.org,2002:null") - tagInt = parser.registerUri("tag:yaml.org,2002:int") - tagFloat = parser.registerUri("tag:yaml.org,2002:float") - events = parser.parse(s) - anchors = initTable[AnchorId, JsonNode]() + levels = newSeq[Level]() + parser = newParser(coreTagLibrary()) + events = parser.parse(s) + anchors = initTable[AnchorId, JsonNode]() for event in events(): case event.kind @@ -56,9 +70,17 @@ proc parseToJson*(s: Stream): seq[JsonNode] = if event.objAnchor != anchorNone: anchors[event.objAnchor] = levels[levels.high].node of yamlScalar: + if levels.len == 0: + # parser ensures that next event will be yamlEndDocument + levels.add((node: jsonFromScalar(event.scalarContent, + event.scalarTag, + event.scalarType), key: nil)) + continue + case levels[levels.high].node.kind of JArray: let jsonScalar = jsonFromScalar(event.scalarContent, + event.scalarTag, event.scalarType) levels[levels.high].node.elems.add(jsonScalar) if event.scalarAnchor != anchorNone: @@ -72,6 +94,7 @@ proc parseToJson*(s: Stream): seq[JsonNode] = "scalar keys may not have anchors in JSON") else: let jsonScalar = jsonFromScalar(event.scalarContent, + event.scalarTag, event.scalarType) levels[levels.high].node.fields.add( (key: levels[levels.high].key, val: jsonScalar)) diff --git a/src/private/lexer.nim b/src/private/lexer.nim index 19854f7..9fdea60 100644 --- a/src/private/lexer.nim +++ b/src/private/lexer.nim @@ -684,7 +684,7 @@ iterator tokens(my: var YamlLexer): YamlLexerToken {.closure.} = of ':', '#': lastSpecialChar = c of '[', ']', '{', '}': - yieldToken(tScalar) + yieldScalarPart() trailingSpace = "" state = ylInitialInLine continue diff --git a/src/private/sequential.nim b/src/private/sequential.nim index d4a56b2..0960514 100644 --- a/src/private/sequential.nim +++ b/src/private/sequential.nim @@ -25,24 +25,11 @@ type BlockScalarStyle = enum bsLiteral, bsFolded -proc newParser*(): YamlSequentialParser = +proc newParser*(tagLib: YamlTagLibrary): YamlSequentialParser = new(result) - result.tags = initOrderedTable[string, TagId]() - result.tags["!"] = tagExclamationMark - result.tags["?"] = tagQuestionMark + result.tagLib = tagLib result.anchors = initOrderedTable[string, AnchorId]() -proc uri*(parser: YamlSequentialParser, id: TagId): string = - for pair in parser.tags.pairs: - if pair[1] == id: - return pair[0] - return nil - -proc registerUri*(parser: var YamlSequentialParser, uri: string): TagId = - result = cast[TagId](parser.tags.len) - if parser.tags.hasKeyOrPut(uri, result): - result = parser.tags[uri] - proc anchor*(parser: YamlSequentialParser, id: AnchorId): string = for pair in parser.anchors.pairs: if pair[1] == id: @@ -106,10 +93,9 @@ proc resolveTag(parser: YamlSequentialParser, tag: var string, result = if quotedString: tagExclamationMark else: tagQuestionMark else: try: - result = parser.tags[tag] + result = parser.tagLib.tags[tag] except KeyError: - result = cast[TagId](parser.tags.len) - parser.tags[tag] = result + result = parser.tagLib.registerUri(tag) tag = "" template yieldScalar(content: string, typeHint: YamlTypeHint, diff --git a/src/private/tagLibrary.nim b/src/private/tagLibrary.nim new file mode 100644 index 0000000..e871850 --- /dev/null +++ b/src/private/tagLibrary.nim @@ -0,0 +1,39 @@ +proc initTagLibrary*(): YamlTagLibrary = + result.tags = initTable[string, TagId]() + result.nextCustomTagId = 1000.TagId + +proc registerUri*(tagLib: var YamlTagLibrary, uri: string): TagId = + tagLib.tags[uri] = tagLib.nextCustomTagId + result = tagLib.nextCustomTagId + tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1) + +proc uri*(tagLib: YamlTagLibrary, id: TagId): string = + for iUri, iId in tagLib.tags.pairs: + if iId == id: + return iUri + raise newException(KeyError, "Unknown tag id: " & $id) + +proc failsafeTagLibrary*(): YamlTagLibrary = + result = initTagLibrary() + result.tags["!"] = tagExclamationMark + result.tags["?"] = tagQuestionMark + result.tags["tag:yaml.org,2002:str"] = tagString + result.tags["tag:yaml.org,2002:seq"] = tagSequence + result.tags["tag:yaml.org,2002:map"] = tagMap + +proc coreTagLibrary*(): YamlTagLibrary = + result = failsafeTagLibrary() + result.tags["tag:yaml.org,2002:null"] = tagNull + result.tags["tag:yaml.org,2002:bool"] = tagBoolean + result.tags["tag:yaml.org,2002:int"] = tagInteger + result.tags["tag:yaml.org,2002:float"] = tagFloat + +proc extendedTagLibrary*(): YamlTagLibrary = + result = coreTagLibrary() + result.tags["tag:yaml.org,2002:omap"] = tagOrderedMap + result.tags["tag:yaml.org,2002:pairs"] = tagPairs + result.tags["tag:yaml.org,2002:binary"] = tagBinary + result.tags["tag:yaml.org,2002:merge"] = tagMerge + result.tags["tag:yaml.org,2002:timestamp"] = tagTimestamp + result.tags["tag:yaml.org,2002:value"] = tagValue + result.tags["tag:yaml.org,2002:yaml"] = tagYaml diff --git a/src/yaml.nim b/src/yaml.nim index 36cf496..2443b14 100644 --- a/src/yaml.nim +++ b/src/yaml.nim @@ -34,14 +34,42 @@ type YamlStream* = iterator(): YamlStreamEvent + YamlTagLibrary* = object + tags: Table[string, TagId] + nextCustomTagId*: TagId + YamlSequentialParser* = ref object - tags: OrderedTable[string, TagId] + tagLib: YamlTagLibrary anchors: OrderedTable[string, AnchorId] const + # failsafe schema + tagExclamationMark*: TagId = 0.TagId # "!" non-specific tag tagQuestionMark* : TagId = 1.TagId # "?" non-specific tag - anchorNone*: AnchorId = (-1).AnchorId # no anchor defined + tagString* : TagId = 2.TagId # !!str tag + tagSequence* : TagId = 3.TagId # !!seq tag + tagMap* : TagId = 4.TagId # !!map tag + + # json & core schema + + tagNull* : TagId = 5.TagId # !!null tag + tagBoolean* : TagId = 6.TagId # !!bool tag + tagInteger* : TagId = 7.TagId # !!int tag + tagFloat* : TagId = 8.TagId # !!float tag + + # other language-independent YAML types (from http://yaml.org/type/ ) + + tagOrderedMap* : TagId = 9.TagId # !!omap tag + tagPairs* : TagId = 10.TagId # !!pairs tag + tagSet* : TagId = 11.TagId # !!set tag + tagBinary* : TagId = 12.TagId # !!binary tag + tagMerge* : TagId = 13.TagId # !!merge tag + tagTimestamp* : TagId = 14.TagId # !!timestamp tag + tagValue* : TagId = 15.TagId # !!value tag + tagYaml* : TagId = 16.TagId # !!yaml tag + + anchorNone*: AnchorId = (-1).AnchorId # no anchor defined # interface @@ -55,11 +83,18 @@ proc `==`*(left, right: AnchorId): bool {.borrow.} proc `$`*(id: AnchorId): string {.borrow.} proc hash*(id: AnchorId): Hash {.borrow.} -proc newParser*(): YamlSequentialParser +proc initTagLibrary*(): YamlTagLibrary +proc registerUri*(tagLib: var YamlTagLibrary, uri: string): TagId +proc uri*(tagLib: YamlTagLibrary, id: TagId): string -proc uri*(parser: YamlSequentialParser, id: TagId): string +# these should be consts, but the Nim VM still has problems handling tables +# properly, so we use constructor procs instead. -proc registerUri*(parser: var YamlSequentialParser, uri: string): TagId +proc failsafeTagLibrary*(): YamlTagLibrary +proc coreTagLibrary*(): YamlTagLibrary +proc extendedTagLibrary*(): YamlTagLibrary + +proc newParser*(tagLib: YamlTagLibrary): YamlSequentialParser proc anchor*(parser: YamlSequentialParser, id: AnchorId): string @@ -71,5 +106,6 @@ proc parseToJson*(s: string): seq[JsonNode] # implementation include private.lexer +include private.tagLibrary include private.sequential -include private.json \ No newline at end of file +include private.json diff --git a/test/parsing.nim b/test/parsing.nim index 17f4563..c56fe4d 100644 --- a/test/parsing.nim +++ b/test/parsing.nim @@ -1,5 +1,5 @@ import "../src/yaml" -import streams +import streams, tables import unittest @@ -100,6 +100,7 @@ proc printDifference(expected, actual: YamlStreamEvent) = template ensure(input: string, expected: varargs[YamlStreamEvent]) {.dirty.} = var + parser = newParser(tagLib) i = 0 events = parser.parse(newStringStream(input)) @@ -118,7 +119,7 @@ template ensure(input: string, expected: varargs[YamlStreamEvent]) {.dirty.} = suite "Parsing": setup: - var parser = newParser() + var tagLib = coreTagLibrary() test "Parsing: Simple Scalar": ensure("Scalar", startDoc(), scalar("Scalar"), endDoc()) @@ -215,46 +216,30 @@ suite "Parsing": test "Parsing: explicit non-specific tag": ensure("! a", startDoc(), scalar("a", tagExclamationMark), endDoc()) test "Parsing: secondary tag handle resolution": - let id = parser.registerUri("tag:yaml.org,2002:str") - ensure("!!str a", startDoc(), scalar("a", id), endDoc()) + ensure("!!str a", startDoc(), scalar("a", tagString), endDoc()) test "Parsing: resolving custom tag handles": - let id = parser.registerUri("tag:example.com,2015:foo") + let fooId = tagLib.registerUri("tag:example.com,2015:foo") ensure("%TAG !t! tag:example.com,2015:\n---\n!t!foo a", startDoc(), - scalar("a", id), endDoc()) + scalar("a", fooId), endDoc()) test "Parsing: tags in sequence": - let - idStr = parser.registerUri("tag:yaml.org,2002:str") - idInt = parser.registerUri("tag:yaml.org,2002:int") ensure(" - !!str a\n - b\n - !!int c\n - d", startDoc(), - startSequence(), scalar("a", idStr), scalar("b"), - scalar("c", idInt), scalar("d"), endSequence(), endDoc()) + startSequence(), scalar("a", tagString), scalar("b"), + scalar("c", tagInteger), scalar("d"), endSequence(), endDoc()) test "Parsing: tags in implicit map": - let - idStr = parser.registerUri("tag:yaml.org,2002:str") - idInt = parser.registerUri("tag:yaml.org,2002:int") ensure("!!str a: b\nc: !!int d\ne: !!str f\ng: h", startDoc(), startMap(), - scalar("a", idStr), scalar("b"), scalar("c"), scalar("d", idInt), - scalar("e"), scalar("f", idStr), scalar("g"), scalar("h"), - endMap(), endDoc()) + scalar("a", tagString), scalar("b"), scalar("c"), + scalar("d", tagInteger), scalar("e"), scalar("f", tagString), + scalar("g"), scalar("h"), endMap(), endDoc()) test "Parsing: tags in explicit map": - let - idStr = parser.registerUri("tag:yaml.org,2002:str") - idInt = parser.registerUri("tag:yaml.org,2002:int") ensure("? !!str a\n: !!int b\n? c\n: !!str d", startDoc(), startMap(), - scalar("a", idStr), scalar("b", idInt), scalar("c"), - scalar("d", idStr), endMap(), endDoc()) + scalar("a", tagString), scalar("b", tagInteger), scalar("c"), + scalar("d", tagString), endMap(), endDoc()) test "Parsing: tags for flow objects": - let - idStr = parser.registerUri("tag:yaml.org,2002:str") - idMap = parser.registerUri("tag:yaml.org,2002:map") - idSeq = parser.registerUri("tag:yaml.org,2002:seq") - ensure("!!map { k: !!seq [ a, !!str b] }", startDoc(), startMap(idMap), - scalar("k"), startSequence(idSeq), scalar("a"), - scalar("b", idStr), endSequence(), endMap(), endDoc()) + ensure("!!map { k: !!seq [ a, !!str b] }", startDoc(), startMap(tagMap), + scalar("k"), startSequence(tagSequence), scalar("a"), + scalar("b", tagString), endSequence(), endMap(), endDoc()) test "Parsing: Tag after directives end": - let - idStr = parser.registerUri("tag:yaml.org,2002:str") - ensure("--- !!str\nfoo", startDoc(), scalar("foo", idStr), endDoc()) + ensure("--- !!str\nfoo", startDoc(), scalar("foo", tagString), endDoc()) test "Parsing: Simple Anchor": ensure("&a str", startDoc(), scalar("str", tagQuestionMark, 0.AnchorId), endDoc()) @@ -271,12 +256,9 @@ suite "Parsing": scalar("d", tagQuestionMark, 1.AnchorId), endMap(), endDoc()) test "Parsing: Anchors and tags": - let - idStr = parser.registerUri("tag:yaml.org,2002:str") - idInt = parser.registerUri("tag:yaml.org,2002:int") ensure(" - &a !!str a\n - !!int b\n - &c !!int c\n - &d d", startDoc(), - startSequence(), scalar("a", idStr, 0.AnchorId), - scalar("b", idInt), scalar("c", idInt, 1.AnchorId), + startSequence(), scalar("a", tagString, 0.AnchorId), + scalar("b", tagInteger), scalar("c", tagInteger, 1.AnchorId), scalar("d", tagQuestionMark, 2.AnchorId), endSequence(), endDoc()) test "Parsing: Aliases in sequence": @@ -297,12 +279,10 @@ suite "Parsing": scalar("c"), alias(1.AnchorId), scalar("d"), endSequence(), endMap(), endDoc()) test "Parsing: Tags on empty scalars": - let - idStr = parser.registerUri("tag:yaml.org,2002:str") - idInt = parser.registerUri("tag:yaml.org,2002:int") ensure("!!str : a\nb: !!int\n!!str : !!str", startDoc(), startMap(), - scalar("", idStr), scalar("a"), scalar("b"), scalar("", idInt), - scalar("", idStr), scalar("", idStr), endMap(), endDoc()) + scalar("", tagString), scalar("a"), scalar("b"), + scalar("", tagInteger), scalar("", tagString), + scalar("", tagString), endMap(), endDoc()) test "Parsing: Anchors on empty scalars": ensure("&a : a\nb: &b\n&c : &a", startDoc(), startMap(), scalar("", tagQuestionMark, 0.AnchorId), scalar("a"),