status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Got rid of yamlError. 

* Replaced yielding yamlError tokens with raising a YamlParserError
 * Renamed sequential.nim to parser.nim because it's the parser (duh)
This commit is contained in:
Felix Krause 2016-01-05 18:14:11 +01:00
parent 8b12d069a5
commit d2baa1749e
10 changed files with 122 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -213,12 +213,8 @@ Output:
- Add type hints for more scalar types
* Parser:
- Properly handle leading spaces in block scalars
- Raise exceptions instead of yielding yamlError tokens, which are a pain to
handle for the caller.
- Add an optional warning callback instead of yielding yamlWarning tokens,
- Add an optional warning callback instead of yielding `yamlWarning` tokens,
which are a pain to handle for the caller.
- Oh and did I mention `yamlError` and `yamlWarning` are evil and need to
die.
* Serialization:
- Support for more standard library types
- Support for ref and ptr types

4
private/events.nim
View File

@ -23,7 +23,7 @@ proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool =
left.scalarType == right.scalarType
of yamlAlias:
result = left.aliasTarget == right.aliasTarget
of yamlError, yamlWarning:
of yamlWarning:
result = left.description == right.description and
left.line == right.line and left.column == right.column
@ -48,7 +48,7 @@ proc `$`*(event: YamlStreamEvent): string =
result &= ", content=\"" & event.scalarContent & '\"'
of yamlAlias:
result &= "aliasTarget=" & $event.aliasTarget
of yamlWarning, yamlError:
of yamlWarning:
result &= "line=" & $event.line & ", column=" & $event.column
result &= ", description=\"" & event.description & '\"'
result &= ")"

3
private/json.nim
View File

@ -141,9 +141,6 @@ proc constructJson*(s: YamlStream): seq[JsonNode] =
of yamlWarning:
echo "YAML warning at line ", event.line, ", column ", event.column,
": ", event.description
of yamlError:
echo "YAML error at line ", event.line, ", column ", event.column,
": ", event.description
of yamlAlias:
# we can savely assume that the alias exists in anchors
# (else the parser would have already thrown an exception)

79
private/sequential.nim → private/parser.nim
View File

@ -67,17 +67,19 @@ template yieldWarning(d: string) {.dirty.} =
yield YamlStreamEvent(kind: yamlWarning, description: d,
line: lex.line, column: lex.column)
template yieldError(d: string) {.dirty.} =
yield YamlStreamEvent(kind: yamlError, description: d,
line: lex.line, column: lex.column)
break parserLoop
template raiseError(message: string) {.dirty.} =
var e = newException(YamlParserError, message)
e.line = lex.line
e.column = lex.column
e.lineContent = lex.getCurrentLine()
raise e
template yieldUnexpectedToken(expected: string = "") {.dirty.} =
var msg = "[" & $state & "] Unexpected token"
if expected.len > 0:
msg.add(" (expected " & expected & ")")
msg.add(": " & $token)
yieldError(msg)
raiseError(msg)
proc resolveAnchor(parser: YamlSequentialParser, anchor: var string):
AnchorId {.inline.} =
@ -113,7 +115,7 @@ template yieldScalar(content: string, typeHint: YamlTypeHint,
"scalar[\"", content, "\", type=", typeHint, "]"
if objectTag.len > 0:
if tag.len > 0:
yieldError("Duplicate tag for scalar (tag=" & tag & ", objectTag=" &
raiseError("Duplicate tag for scalar (tag=" & tag & ", objectTag=" &
objectTag)
tag = objectTag
objectTag = ""
@ -222,13 +224,11 @@ template handleBlockIndicator(expected, possible: openarray[DocumentLevelMode],
level = DocumentLevel(mode: mUnknown, indicatorColumn: -1,
indentationColumn: -1)
else:
yieldError("Invalid token after " & $level.mode)
raiseError("Invalid token after " & $level.mode)
else:
yieldError("Invalid token after " & $level.mode)
raiseError("Invalid token after " & $level.mode)
elif level.mode != mUnknown:
yieldError("Invalid indentation")
elif entering == yamlError:
yieldUnexpectedToken()
raiseError("Invalid indentation")
else:
level.mode = next
level.indicatorColumn = lex.column
@ -265,16 +265,16 @@ template handleTagHandle() {.dirty.} =
if tagShorthands.hasKey(handle):
token = nextToken(lex)
if finished(nextToken):
yieldError("Missing tag suffix")
raiseError("Missing tag suffix")
continue
if token != tTagSuffix:
yieldError("Missing tag suffix")
raiseError("Missing tag suffix")
continue
tag = tagShorthands[handle] & lex.content
if level.indentationColumn == -1 and level.indicatorColumn == -1:
level.indentationColumn = lex.column
else:
yieldError("Unknown tag shorthand: " & handle)
raiseError("Unknown tag shorthand: " & handle)
proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
result = iterator(): YamlStreamEvent =
@ -314,23 +314,22 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
var nextToken = tokens
var token = nextToken(lex)
block parserLoop:
while not finished(nextToken):
case state
of ypInitial:
case token
of tYamlDirective:
if foundYamlDirective:
yieldError("Duplicate %YAML directive")
raiseError("Duplicate %YAML directive")
var
warn = false
actualVersion = ""
for version in [1, 2]:
token = nextToken(lex)
if finished(nextToken):
yieldError("Missing or badly formatted YAML version")
raiseError("Missing or badly formatted YAML version")
if token != tVersionPart:
yieldError("Missing or badly formatted YAML version")
raiseError("Missing or badly formatted YAML version")
if parseInt(lex.content) != version:
warn = true
if actualVersion.len > 0: actualVersion &= "."
@ -342,15 +341,15 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
of tTagDirective:
token = nextToken(lex)
if finished(nextToken):
yieldError("Incomplete %TAG directive")
raiseError("Incomplete %TAG directive")
if token != tTagHandle:
yieldError("Invalid token (expected tag handle)")
raiseError("Invalid token (expected tag handle)")
let tagHandle = lex.content
token = nextToken(lex)
if finished(nextToken):
yieldError("Incomplete %TAG directive")
raiseError("Incomplete %TAG directive")
if token != tTagURI:
yieldError("Invalid token (expected tag URI)")
raiseError("Invalid token (expected tag URI)")
tagShorthands[tagHandle] = lex.content
of tUnknownDirective:
yieldWarning("Unknown directive: " & lex.content)
@ -445,7 +444,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
level.mode = mBlockMapValue
continue
else:
yieldError("Unexpected scalar in " & $level.mode)
raiseError("Unexpected scalar in " & $level.mode)
state = ypBlockAfterScalar
of tScalar:
leaveMoreIndentedLevels()
@ -457,11 +456,11 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
scalarIndentation = lex.column
state = ypBlockAfterScalar
else:
yieldError("Unexpected scalar")
raiseError("Unexpected scalar")
of tAlias:
aliasCache = resolveAlias(parser, lex.content)
if aliasCache == yAnchorNone:
yieldError("[alias] Unknown anchor: " & lex.content)
raiseError("[alias] Unknown anchor: " & lex.content)
if ancestry.len > 0:
if level.mode == mUnknown:
level = ancestry.pop()
@ -472,7 +471,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
of mUnknown, mImplicitBlockMapKey, mBlockSequenceItem:
state = ypBlockAfterAlias
else:
yieldError("Unexpected alias")
raiseError("Unexpected alias")
of tStreamEnd:
closeAllLevels()
yield YamlStreamEvent(kind: yamlEndDocument)
@ -533,7 +532,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
state = ypBlockAfterColon
of tLineStart:
if level.mode == mImplicitBlockMapKey:
yieldError("Missing colon after implicit map key")
raiseError("Missing colon after implicit map key")
if level.mode != mScalar:
yieldScalar(scalarCache, scalarCacheType,
scalarCacheIsQuoted)
@ -570,7 +569,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
state = ypBlockAfterColon
of tLineStart:
if level.mode == mImplicitBlockMapKey:
yieldError("Missing colon after implicit map key")
raiseError("Missing colon after implicit map key")
if level.mode == mUnknown:
assert ancestry.len > 0
level = ancestry.pop()
@ -599,7 +598,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
continue
of tLineStart:
if objectTag.len > 0:
yieldError("Duplicate tag for object")
raiseError("Duplicate tag for object")
else:
objectTag = tag
tag = ""
@ -643,7 +642,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
continue
of tLineStart:
if objectTag.len > 0:
yieldError("Duplicate tag for object")
raiseError("Duplicate tag for object")
else:
objectTag = tag
tag = ""
@ -694,7 +693,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
noAnchor = true
if noAnchor:
# cannot use yield within try/except, so do it here
yieldError("[alias] Unknown anchor: " & lex.content)
raiseError("[alias] Unknown anchor: " & lex.content)
yield YamlStreamEvent(kind: yamlAlias, aliasTarget: aliasCache)
level = ancestry.pop()
state = ypBlockLineEnd
@ -715,17 +714,17 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
case token
of tPlus:
if lineStrip != lsClip:
yieldError("Multiple chomping indicators!")
raiseError("Multiple chomping indicators!")
else:
lineStrip = lsKeep
of tDash:
if lineStrip != lsClip:
yieldError("Multiple chomping indicators!")
raiseError("Multiple chomping indicators!")
else:
lineStrip = lsStrip
of tBlockIndentationIndicator:
if blockScalarIndentation != -1:
yieldError("Multiple indentation indicators!")
raiseError("Multiple indentation indicators!")
else:
blockScalarIndentation = parseInt(lex.content)
of tLineStart:
@ -825,7 +824,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
of mFlowSequenceItem:
yieldScalar("", yTypeUnknown)
else:
yieldError("Internal error! Please report this bug.")
raiseError("Internal error! Please report this bug.")
of tOpeningBrace:
if level.mode != mUnknown:
yieldUnexpectedToken()
@ -908,7 +907,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
of ypFlowAfterTag:
case token
of tTagHandle:
yieldError("Multiple tags on same node!")
raiseError("Multiple tags on same node!")
of tAnchor:
anchor = lex.content
state = ypFlowAfterAnchorAndTag
@ -918,7 +917,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
of ypFlowAfterAnchor:
case token
of tAnchor:
yieldError("Multiple anchors on same node!")
raiseError("Multiple anchors on same node!")
of tTagHandle:
handleTagHandle()
state = ypFlowAfterAnchorAndTag
@ -928,9 +927,9 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
of ypFlowAfterAnchorAndTag:
case token
of tAnchor:
yieldError("Multiple anchors on same node!")
raiseError("Multiple anchors on same node!")
of tTagHandle:
yieldError("Multiple tags on same node!")
raiseError("Multiple tags on same node!")
else:
state = ypFlow
continue
@ -1007,4 +1006,4 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
yieldUnexpectedToken("document end")
token = nextToken(lex)
if token == tError:
yieldError("Lexer error: " & lex.content)
raiseError("Lexer error: " & lex.content)

3
private/presenter.nim
View File

@ -374,9 +374,6 @@ proc present*(s: YamlStream, target: Stream, tagLib: YamlTagLibrary,
cached.enqueue(next)
of yamlWarning:
discard
of yamlError:
raise newException(ValueError, "(" & $item.line & ", " &
$item.column & "): " & item.description)
proc transform*(input: Stream, output: Stream, style: YamlPresentationStyle,
indentationStep: int = 2) =

1
private/tagLibrary.nim
View File

@ -13,7 +13,6 @@ proc registerUri*(tagLib: var YamlTagLibrary, uri: string): TagId =
tagLib.tags[uri] = tagLib.nextCustomTagId
result = tagLib.nextCustomTagId
tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1)
echo "registered ", uri, " as: ", result
proc uri*(tagLib: YamlTagLibrary, id: TagId): string =
for iUri, iId in tagLib.tags.pairs:

12
test/parsing.nim
View File

@ -48,10 +48,7 @@ proc alias(target: AnchorId): YamlStreamEvent =
proc printDifference(expected, actual: YamlStreamEvent) =
if expected.kind != actual.kind:
echo "expected " & $expected.kind & ", got " & $actual.kind
if actual.kind == yamlError:
echo "Error message: (line: ", actual.line, ", column: ",
actual.column, ") ", actual.description
elif actual.kind == yamlWarning:
if actual.kind == yamlWarning:
echo "Warning message: " & actual.description
else:
case expected.kind
@ -106,7 +103,7 @@ template ensure(input: string, expected: varargs[YamlStreamEvent]) {.dirty.} =
parser = newParser(tagLib)
i = 0
events = parser.parse(newStringStream(input))
try:
for token in events():
if i >= expected.len:
echo "received more tokens than expected (next token = ",
@ -119,6 +116,11 @@ template ensure(input: string, expected: varargs[YamlStreamEvent]) {.dirty.} =
fail()
break
i.inc()
except YamlParserError:
let e = cast[YamlParserError](getCurrentException())
echo "Parser error:", getCurrentExceptionMsg()
echo e.lineContent
fail()
suite "Parsing":
setup:

24
test/serializing.nim
View File

@ -11,7 +11,7 @@ suite "Serialization":
setup:
var parser = newParser(serializationTagLibrary)
test "Load string sequence":
test "Serialization: Load string sequence":
let input = newStringStream(" - a\n - b")
var
result: seq[string]
@ -23,13 +23,13 @@ suite "Serialization":
assert result[0] == "a"
assert result[1] == "b"
test "Serialize string sequence":
test "Serialization: Serialize string sequence":
var input = @["a", "b"]
var output = newStringStream()
dump(input, output, ypsBlockOnly, ytsNone)
assert output.data == "%YAML 1.2\n--- \n- a\n- b"
test "Load Table[int, string]":
test "Serialization: Load Table[int, string]":
let input = newStringStream("23: dreiundzwanzig\n42: zweiundvierzig")
var
result: Table[int, string]
@ -41,7 +41,7 @@ suite "Serialization":
assert result[23] == "dreiundzwanzig"
assert result[42] == "zweiundvierzig"
test "Serialize Table[int, string]":
test "Serialization: Serialize Table[int, string]":
var input = initTable[int, string]()
input[23] = "dreiundzwanzig"
input[42] = "zweiundvierzig"
@ -49,7 +49,7 @@ suite "Serialization":
dump(input, output, ypsBlockOnly, ytsNone)
assert output.data == "%YAML 1.2\n--- \n23: dreiundzwanzig\n42: zweiundvierzig"
test "Load Sequences in Sequence":
test "Serialization: Load Sequences in Sequence":
let input = newStringStream(" - [1, 2, 3]\n - [4, 5]\n - [6]")
var
result: seq[seq[int]]
@ -62,13 +62,13 @@ suite "Serialization":
assert result[1] == @[4, 5]
assert result[2] == @[6]
test "Serialize Sequences in Sequence":
test "Serialization: Serialize Sequences in Sequence":
let input = @[@[1, 2, 3], @[4, 5], @[6]]
var output = newStringStream()
dump(input, output, ypsDefault, ytsNone)
assert output.data == "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]"
test "Load custom object":
test "Serialization: Load custom object":
let input = newStringStream("firstname: Peter\nsurname: Pan\nage: 12")
var
result: Person
@ -80,13 +80,13 @@ suite "Serialization":
assert result.surname == "Pan"
assert result.age == 12
test "Serialize custom object":
test "Serialization: Serialize custom object":
let input = Person(firstname: "Peter", surname: "Pan", age: 12)
var output = newStringStream()
dump(input, output, ypsBlockOnly, ytsNone)
assert output.data == "%YAML 1.2\n--- \nfirstname: Peter\nsurname: Pan\nage: 12"
test "Load sequence with explicit tags":
test "Serialization: Load sequence with explicit tags":
let input = newStringStream(
"--- !nim:seq(tag:yaml.org,2002:str)\n- !!str one\n- !!str two")
var
@ -98,13 +98,13 @@ suite "Serialization":
assert result[0] == "one"
assert result[1] == "two"
test "Serialize sequence with explicit tags":
test "Serialization: Serialize sequence with explicit tags":
let input = @["one", "two"]
var output = newStringStream()
dump(input, output, ypsBlockOnly, ytsAll)
assert output.data == "%YAML 1.2\n--- !nim:seq(tag:yaml.org,2002:str) \n- !!str one\n- !!str two"
test "Load custom object with explicit root tag":
test "Serialization: Load custom object with explicit root tag":
let input = newStringStream(
"--- !nim:Person\nfirstname: Peter\nsurname: Pan\nage: 12")
var
@ -117,7 +117,7 @@ suite "Serialization":
assert result.surname == "Pan"
assert result.age == 12
test "Serialize custom object with explicit root tag":
test "Serialization: Serialize custom object with explicit root tag":
let input = Person(firstname: "Peter", surname: "Pan", age: 12)
var output = newStringStream()
dump(input, output, ypsBlockOnly, ytsRootOnly)

52
yaml.nim
View File

@ -50,8 +50,7 @@ type
## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds
## are discussed in ``YamlStreamEvent``.
yamlStartDocument, yamlEndDocument, yamlStartMap, yamlEndMap,
yamlStartSequence, yamlEndSequence, yamlScalar, yamlAlias,
yamlError, yamlWarning
yamlStartSequence, yamlEndSequence, yamlScalar, yamlAlias, yamlWarning
TagId* = distinct int ## \
## A ``TagId`` identifies a tag URI, like for example
@ -62,6 +61,7 @@ type
## mapped to ``TagId`` s for efficiency reasons (you do not need to
## compare strings every time) and to be able to discover unknown tag
## URIs early in the parsing process.
AnchorId* = distinct int ## \
## An ``AnchorId`` identifies an anchor in the current document. It
## becomes invalid as soon as the current document scope is invalidated
@ -107,7 +107,7 @@ type
discard
of yamlAlias:
aliasTarget* : AnchorId
of yamlError, yamlWarning:
of yamlWarning:
description* : string
line* : int
column* : int
@ -185,6 +185,47 @@ type
## - ``ypsBlockOnly``: Formats all output in block style, does not use
## flow style at all.
ypsMinimal, ypsCanonical, ypsDefault, ypsJson, ypsBlockOnly
YamlParserError* = object of Exception
## A parser error is raised if the character stream that is parsed is
## not a valid YAML character stream. This stream cannot and will not be
## parsed wholly nor partially and all events that have been emitted by
## the YamlStream the parser provides should be discarded.
##
## A character stream is invalid YAML if and only if at least one of the
## following conditions apply:
##
## - There are invalid characters in an element whose contents is
## restricted to a limited set of characters. For example, there are
## characters in a tag URI which are not valid URI characters.
## - An element has invalid indentation. This can happen for example if
## a block list element indicated by ``"- "`` is less indented than
## the element in the previous line, but there is no block sequence
## list open at the same indentation level.
## - The YAML structure is invalid. For example, an explicit block map
## indicated by ``"? "`` and ``": "`` may not suddenly have a block
## sequence item (``"- "``) at the same indentation level. Another
## possible violation is closing a flow style object with the wrong
## closing character (``}``, ``]``) or not closing it at all.
## - A custom tag shorthand is used that has not previously been
## declared with a ``%TAG`` directive.
## - Multiple tags or anchors are defined for the same node.
## - An alias is used which does not map to any anchor that has
## previously been declared in the same document.
## - An alias has a tag or anchor associated with it.
##
## Some elements in this list are vague. For a detailed description of a
## valid YAML character stream, see the YAML specification.
line*: int ## line number (1-based) where the error was encountered
column*: int ## \
## column number (1-based) where the error was encountered
lineContent*: string ## \
## content of the line where the error was encountered. Includes a
## second line with a marker ``^`` at the position where the error
## was encountered, as returned by ``lexbase.getCurrentLine``.
YamlPresenterError* = object of Exception
## Exception that may be raised by the YAML presenter.
const
# failsafe schema
@ -312,7 +353,8 @@ proc newParser*(tagLib: YamlTagLibrary): YamlSequentialParser
proc anchor*(parser: YamlSequentialParser, id: AnchorId): string
## Get the anchor name which an ``AnchorId`` maps to
proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream
proc parse*(parser: YamlSequentialParser, s: Stream):
YamlStream {.raises: [IOError, YamlParserError].}
## Parse a YAML character stream. ``s`` must be readable.
proc constructJson*(s: YamlStream): seq[JsonNode]
@ -344,6 +386,6 @@ proc transform*(input: Stream, output: Stream, style: YamlPresentationStyle,
include private.lexer
include private.tagLibrary
include private.events
include private.sequential
include private.parser
include private.json
include private.presenter

4
yaml/serialization.nim
View File

@ -129,7 +129,6 @@ macro make_serializable*(types: stmt): stmt =
if finished(s):
raise newException(ValueError, "Construction error!")
while event.kind != yamlEndMap:
if event.kind == yamlError: echo event.description
assert event.kind == yamlScalar
assert event.scalarTag in [yTagQuestionMark, yTagString]
case hash(event.scalarContent)
@ -139,7 +138,7 @@ macro make_serializable*(types: stmt): stmt =
event = s()
if finished(s):
raise newException(ValueError, "Construction error!")
var keyCase = impl[5][1][3]
var keyCase = impl[5][1][2]
assert keyCase.kind == nnkCaseStmt
for field in objectFields(recList):
let nameHash = hash($field.name.ident)
@ -215,7 +214,6 @@ macro make_serializable*(types: stmt): stmt =
nnkIteratorDef)))
serializeProc[6] = impl
result.add(serializeProc)
echo result.repr
proc prepend*(event: YamlStreamEvent, s: YamlStream): YamlStream =
result = iterator(): YamlStreamEvent =