status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Worked on tag URI interface 

* Added YamlTagLibrary
 * adhere to explicit tags in JSON
 * Fixed lexer type hints before YAML flow special characters
This commit is contained in:
Felix Krause 2015-12-26 18:40:23 +01:00
parent ca0f5d1741
commit 075cf8b8ef
6 changed files with 142 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/private/json.nim
View File

@ -3,9 +3,28 @@ type
proc initLevel(node: JsonNode): Level = (node: node, key: cast[string](nil))
proc jsonFromScalar(content: string, typeHint: YamlTypeHint): JsonNode =
proc jsonFromScalar(content: string, tag: TagId,
typeHint: YamlTypeHint): JsonNode =
new(result)
case typeHint
var mappedType: YamlTypeHint
case tag
of tagQuestionMark:
mappedType = typeHint
of tagExclamationMark, tagString:
mappedType = yTypeString
of tagBoolean:
mappedType = yTypeBoolean
of tagInteger:
mappedType = yTypeInteger
of tagNull:
mappedType = yTypeNull
of tagFloat:
mappedType = yTypeFloat
else:
mappedType = yTypeUnknown
case mappedType
of yTypeInteger:
result.kind = JInt
result.num = parseBiggestInt(content)
@ -28,15 +47,10 @@ proc parseToJson*(s: Stream): seq[JsonNode] =
newSeq(result, 0)
var
levels = newSeq[Level]()
parser = newParser()
tagStr = parser.registerUri("tag:yaml.org,2002:str")
tagBool = parser.registerUri("tag:yaml.org,2002:bool")
tagNull = parser.registerUri("tag:yaml.org,2002:null")
tagInt = parser.registerUri("tag:yaml.org,2002:int")
tagFloat = parser.registerUri("tag:yaml.org,2002:float")
events = parser.parse(s)
anchors = initTable[AnchorId, JsonNode]()
levels = newSeq[Level]()
parser = newParser(coreTagLibrary())
events = parser.parse(s)
anchors = initTable[AnchorId, JsonNode]()
for event in events():
case event.kind
@ -56,9 +70,17 @@ proc parseToJson*(s: Stream): seq[JsonNode] =
if event.objAnchor != anchorNone:
anchors[event.objAnchor] = levels[levels.high].node
of yamlScalar:
if levels.len == 0:
# parser ensures that next event will be yamlEndDocument
levels.add((node: jsonFromScalar(event.scalarContent,
event.scalarTag,
event.scalarType), key: nil))
continue
case levels[levels.high].node.kind
of JArray:
let jsonScalar = jsonFromScalar(event.scalarContent,
event.scalarTag,
event.scalarType)
levels[levels.high].node.elems.add(jsonScalar)
if event.scalarAnchor != anchorNone:
@ -72,6 +94,7 @@ proc parseToJson*(s: Stream): seq[JsonNode] =
"scalar keys may not have anchors in JSON")
else:
let jsonScalar = jsonFromScalar(event.scalarContent,
event.scalarTag,
event.scalarType)
levels[levels.high].node.fields.add(
(key: levels[levels.high].key, val: jsonScalar))

2
src/private/lexer.nim
View File

@ -684,7 +684,7 @@ iterator tokens(my: var YamlLexer): YamlLexerToken {.closure.} =
of ':', '#':
lastSpecialChar = c
of '[', ']', '{', '}':
yieldToken(tScalar)
yieldScalarPart()
trailingSpace = ""
state = ylInitialInLine
continue

22
src/private/sequential.nim
View File

@ -25,24 +25,11 @@ type
BlockScalarStyle = enum
bsLiteral, bsFolded
proc newParser*(): YamlSequentialParser =
proc newParser*(tagLib: YamlTagLibrary): YamlSequentialParser =
new(result)
result.tags = initOrderedTable[string, TagId]()
result.tags["!"] = tagExclamationMark
result.tags["?"] = tagQuestionMark
result.tagLib = tagLib
result.anchors = initOrderedTable[string, AnchorId]()
proc uri*(parser: YamlSequentialParser, id: TagId): string =
for pair in parser.tags.pairs:
if pair[1] == id:
return pair[0]
return nil
proc registerUri*(parser: var YamlSequentialParser, uri: string): TagId =
result = cast[TagId](parser.tags.len)
if parser.tags.hasKeyOrPut(uri, result):
result = parser.tags[uri]
proc anchor*(parser: YamlSequentialParser, id: AnchorId): string =
for pair in parser.anchors.pairs:
if pair[1] == id:
@ -106,10 +93,9 @@ proc resolveTag(parser: YamlSequentialParser, tag: var string,
result = if quotedString: tagExclamationMark else: tagQuestionMark
else:
try:
result = parser.tags[tag]
result = parser.tagLib.tags[tag]
except KeyError:
result = cast[TagId](parser.tags.len)
parser.tags[tag] = result
result = parser.tagLib.registerUri(tag)
tag = ""
template yieldScalar(content: string, typeHint: YamlTypeHint,

39
src/private/tagLibrary.nim Normal file
View File

@ -0,0 +1,39 @@
proc initTagLibrary*(): YamlTagLibrary =
result.tags = initTable[string, TagId]()
result.nextCustomTagId = 1000.TagId
proc registerUri*(tagLib: var YamlTagLibrary, uri: string): TagId =
tagLib.tags[uri] = tagLib.nextCustomTagId
result = tagLib.nextCustomTagId
tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1)
proc uri*(tagLib: YamlTagLibrary, id: TagId): string =
for iUri, iId in tagLib.tags.pairs:
if iId == id:
return iUri
raise newException(KeyError, "Unknown tag id: " & $id)
proc failsafeTagLibrary*(): YamlTagLibrary =
result = initTagLibrary()
result.tags["!"] = tagExclamationMark
result.tags["?"] = tagQuestionMark
result.tags["tag:yaml.org,2002:str"] = tagString
result.tags["tag:yaml.org,2002:seq"] = tagSequence
result.tags["tag:yaml.org,2002:map"] = tagMap
proc coreTagLibrary*(): YamlTagLibrary =
result = failsafeTagLibrary()
result.tags["tag:yaml.org,2002:null"] = tagNull
result.tags["tag:yaml.org,2002:bool"] = tagBoolean
result.tags["tag:yaml.org,2002:int"] = tagInteger
result.tags["tag:yaml.org,2002:float"] = tagFloat
proc extendedTagLibrary*(): YamlTagLibrary =
result = coreTagLibrary()
result.tags["tag:yaml.org,2002:omap"] = tagOrderedMap
result.tags["tag:yaml.org,2002:pairs"] = tagPairs
result.tags["tag:yaml.org,2002:binary"] = tagBinary
result.tags["tag:yaml.org,2002:merge"] = tagMerge
result.tags["tag:yaml.org,2002:timestamp"] = tagTimestamp
result.tags["tag:yaml.org,2002:value"] = tagValue
result.tags["tag:yaml.org,2002:yaml"] = tagYaml

48
src/yaml.nim
View File

@ -34,14 +34,42 @@ type
YamlStream* = iterator(): YamlStreamEvent
YamlTagLibrary* = object
tags: Table[string, TagId]
nextCustomTagId*: TagId
YamlSequentialParser* = ref object
tags: OrderedTable[string, TagId]
tagLib: YamlTagLibrary
anchors: OrderedTable[string, AnchorId]
const
# failsafe schema
tagExclamationMark*: TagId = 0.TagId # "!" non-specific tag
tagQuestionMark* : TagId = 1.TagId # "?" non-specific tag
anchorNone*: AnchorId = (-1).AnchorId # no anchor defined
tagString* : TagId = 2.TagId # !!str tag
tagSequence* : TagId = 3.TagId # !!seq tag
tagMap* : TagId = 4.TagId # !!map tag
# json & core schema
tagNull* : TagId = 5.TagId # !!null tag
tagBoolean* : TagId = 6.TagId # !!bool tag
tagInteger* : TagId = 7.TagId # !!int tag
tagFloat* : TagId = 8.TagId # !!float tag
# other language-independent YAML types (from http://yaml.org/type/ )
tagOrderedMap* : TagId = 9.TagId # !!omap tag
tagPairs* : TagId = 10.TagId # !!pairs tag
tagSet* : TagId = 11.TagId # !!set tag
tagBinary* : TagId = 12.TagId # !!binary tag
tagMerge* : TagId = 13.TagId # !!merge tag
tagTimestamp* : TagId = 14.TagId # !!timestamp tag
tagValue* : TagId = 15.TagId # !!value tag
tagYaml* : TagId = 16.TagId # !!yaml tag
anchorNone*: AnchorId = (-1).AnchorId # no anchor defined
# interface
@ -55,11 +83,18 @@ proc `==`*(left, right: AnchorId): bool {.borrow.}
proc `$`*(id: AnchorId): string {.borrow.}
proc hash*(id: AnchorId): Hash {.borrow.}
proc newParser*(): YamlSequentialParser
proc initTagLibrary*(): YamlTagLibrary
proc registerUri*(tagLib: var YamlTagLibrary, uri: string): TagId
proc uri*(tagLib: YamlTagLibrary, id: TagId): string
proc uri*(parser: YamlSequentialParser, id: TagId): string
# these should be consts, but the Nim VM still has problems handling tables
# properly, so we use constructor procs instead.
proc registerUri*(parser: var YamlSequentialParser, uri: string): TagId
proc failsafeTagLibrary*(): YamlTagLibrary
proc coreTagLibrary*(): YamlTagLibrary
proc extendedTagLibrary*(): YamlTagLibrary
proc newParser*(tagLib: YamlTagLibrary): YamlSequentialParser
proc anchor*(parser: YamlSequentialParser, id: AnchorId): string
@ -71,5 +106,6 @@ proc parseToJson*(s: string): seq[JsonNode]
# implementation
include private.lexer
include private.tagLibrary
include private.sequential
include private.json
include private.json

64
test/parsing.nim
View File

@ -1,5 +1,5 @@
import "../src/yaml"
import streams
import streams, tables
import unittest
@ -100,6 +100,7 @@ proc printDifference(expected, actual: YamlStreamEvent) =
template ensure(input: string, expected: varargs[YamlStreamEvent]) {.dirty.} =
var
parser = newParser(tagLib)
i = 0
events = parser.parse(newStringStream(input))
@ -118,7 +119,7 @@ template ensure(input: string, expected: varargs[YamlStreamEvent]) {.dirty.} =
suite "Parsing":
setup:
var parser = newParser()
var tagLib = coreTagLibrary()
test "Parsing: Simple Scalar":
ensure("Scalar", startDoc(), scalar("Scalar"), endDoc())
@ -215,46 +216,30 @@ suite "Parsing":
test "Parsing: explicit non-specific tag":
ensure("! a", startDoc(), scalar("a", tagExclamationMark), endDoc())
test "Parsing: secondary tag handle resolution":
let id = parser.registerUri("tag:yaml.org,2002:str")
ensure("!!str a", startDoc(), scalar("a", id), endDoc())
ensure("!!str a", startDoc(), scalar("a", tagString), endDoc())
test "Parsing: resolving custom tag handles":
let id = parser.registerUri("tag:example.com,2015:foo")
let fooId = tagLib.registerUri("tag:example.com,2015:foo")
ensure("%TAG !t! tag:example.com,2015:\n---\n!t!foo a", startDoc(),
scalar("a", id), endDoc())
scalar("a", fooId), endDoc())
test "Parsing: tags in sequence":
let
idStr = parser.registerUri("tag:yaml.org,2002:str")
idInt = parser.registerUri("tag:yaml.org,2002:int")
ensure(" - !!str a\n - b\n - !!int c\n - d", startDoc(),
startSequence(), scalar("a", idStr), scalar("b"),
scalar("c", idInt), scalar("d"), endSequence(), endDoc())
startSequence(), scalar("a", tagString), scalar("b"),
scalar("c", tagInteger), scalar("d"), endSequence(), endDoc())
test "Parsing: tags in implicit map":
let
idStr = parser.registerUri("tag:yaml.org,2002:str")
idInt = parser.registerUri("tag:yaml.org,2002:int")
ensure("!!str a: b\nc: !!int d\ne: !!str f\ng: h", startDoc(), startMap(),
scalar("a", idStr), scalar("b"), scalar("c"), scalar("d", idInt),
scalar("e"), scalar("f", idStr), scalar("g"), scalar("h"),
endMap(), endDoc())
scalar("a", tagString), scalar("b"), scalar("c"),
scalar("d", tagInteger), scalar("e"), scalar("f", tagString),
scalar("g"), scalar("h"), endMap(), endDoc())
test "Parsing: tags in explicit map":
let
idStr = parser.registerUri("tag:yaml.org,2002:str")
idInt = parser.registerUri("tag:yaml.org,2002:int")
ensure("? !!str a\n: !!int b\n? c\n: !!str d", startDoc(), startMap(),
scalar("a", idStr), scalar("b", idInt), scalar("c"),
scalar("d", idStr), endMap(), endDoc())
scalar("a", tagString), scalar("b", tagInteger), scalar("c"),
scalar("d", tagString), endMap(), endDoc())
test "Parsing: tags for flow objects":
let
idStr = parser.registerUri("tag:yaml.org,2002:str")
idMap = parser.registerUri("tag:yaml.org,2002:map")
idSeq = parser.registerUri("tag:yaml.org,2002:seq")
ensure("!!map { k: !!seq [ a, !!str b] }", startDoc(), startMap(idMap),
scalar("k"), startSequence(idSeq), scalar("a"),
scalar("b", idStr), endSequence(), endMap(), endDoc())
ensure("!!map { k: !!seq [ a, !!str b] }", startDoc(), startMap(tagMap),
scalar("k"), startSequence(tagSequence), scalar("a"),
scalar("b", tagString), endSequence(), endMap(), endDoc())
test "Parsing: Tag after directives end":
let
idStr = parser.registerUri("tag:yaml.org,2002:str")
ensure("--- !!str\nfoo", startDoc(), scalar("foo", idStr), endDoc())
ensure("--- !!str\nfoo", startDoc(), scalar("foo", tagString), endDoc())
test "Parsing: Simple Anchor":
ensure("&a str", startDoc(), scalar("str", tagQuestionMark,
0.AnchorId), endDoc())
@ -271,12 +256,9 @@ suite "Parsing":
scalar("d", tagQuestionMark, 1.AnchorId),
endMap(), endDoc())
test "Parsing: Anchors and tags":
let
idStr = parser.registerUri("tag:yaml.org,2002:str")
idInt = parser.registerUri("tag:yaml.org,2002:int")
ensure(" - &a !!str a\n - !!int b\n - &c !!int c\n - &d d", startDoc(),
startSequence(), scalar("a", idStr, 0.AnchorId),
scalar("b", idInt), scalar("c", idInt, 1.AnchorId),
startSequence(), scalar("a", tagString, 0.AnchorId),
scalar("b", tagInteger), scalar("c", tagInteger, 1.AnchorId),
scalar("d", tagQuestionMark, 2.AnchorId), endSequence(),
endDoc())
test "Parsing: Aliases in sequence":
@ -297,12 +279,10 @@ suite "Parsing":
scalar("c"), alias(1.AnchorId), scalar("d"), endSequence(),
endMap(), endDoc())
test "Parsing: Tags on empty scalars":
let
idStr = parser.registerUri("tag:yaml.org,2002:str")
idInt = parser.registerUri("tag:yaml.org,2002:int")
ensure("!!str : a\nb: !!int\n!!str : !!str", startDoc(), startMap(),
scalar("", idStr), scalar("a"), scalar("b"), scalar("", idInt),
scalar("", idStr), scalar("", idStr), endMap(), endDoc())
scalar("", tagString), scalar("a"), scalar("b"),
scalar("", tagInteger), scalar("", tagString),
scalar("", tagString), endMap(), endDoc())
test "Parsing: Anchors on empty scalars":
ensure("&a : a\nb: &b\n&c : &a", startDoc(), startMap(),
scalar("", tagQuestionMark, 0.AnchorId), scalar("a"),