diff --git a/README.md b/README.md index 5b4e1ce..ef09c95 100644 --- a/README.md +++ b/README.md @@ -213,12 +213,8 @@ Output: - Add type hints for more scalar types * Parser: - Properly handle leading spaces in block scalars - - Raise exceptions instead of yielding yamlError tokens, which are a pain to - handle for the caller. - - Add an optional warning callback instead of yielding yamlWarning tokens, + - Add an optional warning callback instead of yielding `yamlWarning` tokens, which are a pain to handle for the caller. - - Oh and did I mention `yamlError` and `yamlWarning` are evil and need to - die. * Serialization: - Support for more standard library types - Support for ref and ptr types diff --git a/private/events.nim b/private/events.nim index 6f6f713..35b0206 100644 --- a/private/events.nim +++ b/private/events.nim @@ -23,7 +23,7 @@ proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool = left.scalarType == right.scalarType of yamlAlias: result = left.aliasTarget == right.aliasTarget - of yamlError, yamlWarning: + of yamlWarning: result = left.description == right.description and left.line == right.line and left.column == right.column @@ -48,7 +48,7 @@ proc `$`*(event: YamlStreamEvent): string = result &= ", content=\"" & event.scalarContent & '\"' of yamlAlias: result &= "aliasTarget=" & $event.aliasTarget - of yamlWarning, yamlError: + of yamlWarning: result &= "line=" & $event.line & ", column=" & $event.column result &= ", description=\"" & event.description & '\"' result &= ")" diff --git a/private/json.nim b/private/json.nim index f49346f..c75be8d 100644 --- a/private/json.nim +++ b/private/json.nim @@ -141,9 +141,6 @@ proc constructJson*(s: YamlStream): seq[JsonNode] = of yamlWarning: echo "YAML warning at line ", event.line, ", column ", event.column, ": ", event.description - of yamlError: - echo "YAML error at line ", event.line, ", column ", event.column, - ": ", event.description of yamlAlias: # we can savely assume that the alias exists in anchors # (else the parser would have already thrown an exception) diff --git a/private/sequential.nim b/private/parser.nim similarity index 94% rename from private/sequential.nim rename to private/parser.nim index 5913aa2..fbe607b 100644 --- a/private/sequential.nim +++ b/private/parser.nim @@ -67,17 +67,19 @@ template yieldWarning(d: string) {.dirty.} = yield YamlStreamEvent(kind: yamlWarning, description: d, line: lex.line, column: lex.column) -template yieldError(d: string) {.dirty.} = - yield YamlStreamEvent(kind: yamlError, description: d, - line: lex.line, column: lex.column) - break parserLoop +template raiseError(message: string) {.dirty.} = + var e = newException(YamlParserError, message) + e.line = lex.line + e.column = lex.column + e.lineContent = lex.getCurrentLine() + raise e template yieldUnexpectedToken(expected: string = "") {.dirty.} = var msg = "[" & $state & "] Unexpected token" if expected.len > 0: msg.add(" (expected " & expected & ")") msg.add(": " & $token) - yieldError(msg) + raiseError(msg) proc resolveAnchor(parser: YamlSequentialParser, anchor: var string): AnchorId {.inline.} = @@ -113,7 +115,7 @@ template yieldScalar(content: string, typeHint: YamlTypeHint, "scalar[\"", content, "\", type=", typeHint, "]" if objectTag.len > 0: if tag.len > 0: - yieldError("Duplicate tag for scalar (tag=" & tag & ", objectTag=" & + raiseError("Duplicate tag for scalar (tag=" & tag & ", objectTag=" & objectTag) tag = objectTag objectTag = "" @@ -222,13 +224,11 @@ template handleBlockIndicator(expected, possible: openarray[DocumentLevelMode], level = DocumentLevel(mode: mUnknown, indicatorColumn: -1, indentationColumn: -1) else: - yieldError("Invalid token after " & $level.mode) + raiseError("Invalid token after " & $level.mode) else: - yieldError("Invalid token after " & $level.mode) + raiseError("Invalid token after " & $level.mode) elif level.mode != mUnknown: - yieldError("Invalid indentation") - elif entering == yamlError: - yieldUnexpectedToken() + raiseError("Invalid indentation") else: level.mode = next level.indicatorColumn = lex.column @@ -265,16 +265,16 @@ template handleTagHandle() {.dirty.} = if tagShorthands.hasKey(handle): token = nextToken(lex) if finished(nextToken): - yieldError("Missing tag suffix") + raiseError("Missing tag suffix") continue if token != tTagSuffix: - yieldError("Missing tag suffix") + raiseError("Missing tag suffix") continue tag = tagShorthands[handle] & lex.content if level.indentationColumn == -1 and level.indicatorColumn == -1: level.indentationColumn = lex.column else: - yieldError("Unknown tag shorthand: " & handle) + raiseError("Unknown tag shorthand: " & handle) proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = result = iterator(): YamlStreamEvent = @@ -314,23 +314,22 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = var nextToken = tokens var token = nextToken(lex) - block parserLoop: - while not finished(nextToken): + while not finished(nextToken): case state of ypInitial: case token of tYamlDirective: if foundYamlDirective: - yieldError("Duplicate %YAML directive") + raiseError("Duplicate %YAML directive") var warn = false actualVersion = "" for version in [1, 2]: token = nextToken(lex) if finished(nextToken): - yieldError("Missing or badly formatted YAML version") + raiseError("Missing or badly formatted YAML version") if token != tVersionPart: - yieldError("Missing or badly formatted YAML version") + raiseError("Missing or badly formatted YAML version") if parseInt(lex.content) != version: warn = true if actualVersion.len > 0: actualVersion &= "." @@ -342,15 +341,15 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = of tTagDirective: token = nextToken(lex) if finished(nextToken): - yieldError("Incomplete %TAG directive") + raiseError("Incomplete %TAG directive") if token != tTagHandle: - yieldError("Invalid token (expected tag handle)") + raiseError("Invalid token (expected tag handle)") let tagHandle = lex.content token = nextToken(lex) if finished(nextToken): - yieldError("Incomplete %TAG directive") + raiseError("Incomplete %TAG directive") if token != tTagURI: - yieldError("Invalid token (expected tag URI)") + raiseError("Invalid token (expected tag URI)") tagShorthands[tagHandle] = lex.content of tUnknownDirective: yieldWarning("Unknown directive: " & lex.content) @@ -445,7 +444,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = level.mode = mBlockMapValue continue else: - yieldError("Unexpected scalar in " & $level.mode) + raiseError("Unexpected scalar in " & $level.mode) state = ypBlockAfterScalar of tScalar: leaveMoreIndentedLevels() @@ -457,11 +456,11 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = scalarIndentation = lex.column state = ypBlockAfterScalar else: - yieldError("Unexpected scalar") + raiseError("Unexpected scalar") of tAlias: aliasCache = resolveAlias(parser, lex.content) if aliasCache == yAnchorNone: - yieldError("[alias] Unknown anchor: " & lex.content) + raiseError("[alias] Unknown anchor: " & lex.content) if ancestry.len > 0: if level.mode == mUnknown: level = ancestry.pop() @@ -472,7 +471,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = of mUnknown, mImplicitBlockMapKey, mBlockSequenceItem: state = ypBlockAfterAlias else: - yieldError("Unexpected alias") + raiseError("Unexpected alias") of tStreamEnd: closeAllLevels() yield YamlStreamEvent(kind: yamlEndDocument) @@ -533,7 +532,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = state = ypBlockAfterColon of tLineStart: if level.mode == mImplicitBlockMapKey: - yieldError("Missing colon after implicit map key") + raiseError("Missing colon after implicit map key") if level.mode != mScalar: yieldScalar(scalarCache, scalarCacheType, scalarCacheIsQuoted) @@ -570,7 +569,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = state = ypBlockAfterColon of tLineStart: if level.mode == mImplicitBlockMapKey: - yieldError("Missing colon after implicit map key") + raiseError("Missing colon after implicit map key") if level.mode == mUnknown: assert ancestry.len > 0 level = ancestry.pop() @@ -599,7 +598,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = continue of tLineStart: if objectTag.len > 0: - yieldError("Duplicate tag for object") + raiseError("Duplicate tag for object") else: objectTag = tag tag = "" @@ -643,7 +642,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = continue of tLineStart: if objectTag.len > 0: - yieldError("Duplicate tag for object") + raiseError("Duplicate tag for object") else: objectTag = tag tag = "" @@ -694,7 +693,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = noAnchor = true if noAnchor: # cannot use yield within try/except, so do it here - yieldError("[alias] Unknown anchor: " & lex.content) + raiseError("[alias] Unknown anchor: " & lex.content) yield YamlStreamEvent(kind: yamlAlias, aliasTarget: aliasCache) level = ancestry.pop() state = ypBlockLineEnd @@ -715,17 +714,17 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = case token of tPlus: if lineStrip != lsClip: - yieldError("Multiple chomping indicators!") + raiseError("Multiple chomping indicators!") else: lineStrip = lsKeep of tDash: if lineStrip != lsClip: - yieldError("Multiple chomping indicators!") + raiseError("Multiple chomping indicators!") else: lineStrip = lsStrip of tBlockIndentationIndicator: if blockScalarIndentation != -1: - yieldError("Multiple indentation indicators!") + raiseError("Multiple indentation indicators!") else: blockScalarIndentation = parseInt(lex.content) of tLineStart: @@ -825,7 +824,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = of mFlowSequenceItem: yieldScalar("", yTypeUnknown) else: - yieldError("Internal error! Please report this bug.") + raiseError("Internal error! Please report this bug.") of tOpeningBrace: if level.mode != mUnknown: yieldUnexpectedToken() @@ -908,7 +907,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = of ypFlowAfterTag: case token of tTagHandle: - yieldError("Multiple tags on same node!") + raiseError("Multiple tags on same node!") of tAnchor: anchor = lex.content state = ypFlowAfterAnchorAndTag @@ -918,7 +917,7 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = of ypFlowAfterAnchor: case token of tAnchor: - yieldError("Multiple anchors on same node!") + raiseError("Multiple anchors on same node!") of tTagHandle: handleTagHandle() state = ypFlowAfterAnchorAndTag @@ -928,9 +927,9 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = of ypFlowAfterAnchorAndTag: case token of tAnchor: - yieldError("Multiple anchors on same node!") + raiseError("Multiple anchors on same node!") of tTagHandle: - yieldError("Multiple tags on same node!") + raiseError("Multiple tags on same node!") else: state = ypFlow continue @@ -1007,4 +1006,4 @@ proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream = yieldUnexpectedToken("document end") token = nextToken(lex) if token == tError: - yieldError("Lexer error: " & lex.content) \ No newline at end of file + raiseError("Lexer error: " & lex.content) \ No newline at end of file diff --git a/private/presenter.nim b/private/presenter.nim index 8ad6e2b..0af03ae 100644 --- a/private/presenter.nim +++ b/private/presenter.nim @@ -374,9 +374,6 @@ proc present*(s: YamlStream, target: Stream, tagLib: YamlTagLibrary, cached.enqueue(next) of yamlWarning: discard - of yamlError: - raise newException(ValueError, "(" & $item.line & ", " & - $item.column & "): " & item.description) proc transform*(input: Stream, output: Stream, style: YamlPresentationStyle, indentationStep: int = 2) = diff --git a/private/tagLibrary.nim b/private/tagLibrary.nim index ddabd2e..42c6600 100644 --- a/private/tagLibrary.nim +++ b/private/tagLibrary.nim @@ -13,7 +13,6 @@ proc registerUri*(tagLib: var YamlTagLibrary, uri: string): TagId = tagLib.tags[uri] = tagLib.nextCustomTagId result = tagLib.nextCustomTagId tagLib.nextCustomTagId = cast[TagId](cast[int](tagLib.nextCustomTagId) + 1) - echo "registered ", uri, " as: ", result proc uri*(tagLib: YamlTagLibrary, id: TagId): string = for iUri, iId in tagLib.tags.pairs: diff --git a/test/parsing.nim b/test/parsing.nim index 4f0cac8..47d56fc 100644 --- a/test/parsing.nim +++ b/test/parsing.nim @@ -48,10 +48,7 @@ proc alias(target: AnchorId): YamlStreamEvent = proc printDifference(expected, actual: YamlStreamEvent) = if expected.kind != actual.kind: echo "expected " & $expected.kind & ", got " & $actual.kind - if actual.kind == yamlError: - echo "Error message: (line: ", actual.line, ", column: ", - actual.column, ") ", actual.description - elif actual.kind == yamlWarning: + if actual.kind == yamlWarning: echo "Warning message: " & actual.description else: case expected.kind @@ -106,19 +103,24 @@ template ensure(input: string, expected: varargs[YamlStreamEvent]) {.dirty.} = parser = newParser(tagLib) i = 0 events = parser.parse(newStringStream(input)) - - for token in events(): - if i >= expected.len: - echo "received more tokens than expected (next token = ", - token.kind, ")" - fail() - break - if token != expected[i]: - echo "at token #" & $i & ":" - printDifference(expected[i], token) - fail() - break - i.inc() + try: + for token in events(): + if i >= expected.len: + echo "received more tokens than expected (next token = ", + token.kind, ")" + fail() + break + if token != expected[i]: + echo "at token #" & $i & ":" + printDifference(expected[i], token) + fail() + break + i.inc() + except YamlParserError: + let e = cast[YamlParserError](getCurrentException()) + echo "Parser error:", getCurrentExceptionMsg() + echo e.lineContent + fail() suite "Parsing": setup: diff --git a/test/serializing.nim b/test/serializing.nim index ecff54c..e3ca115 100644 --- a/test/serializing.nim +++ b/test/serializing.nim @@ -11,7 +11,7 @@ suite "Serialization": setup: var parser = newParser(serializationTagLibrary) - test "Load string sequence": + test "Serialization: Load string sequence": let input = newStringStream(" - a\n - b") var result: seq[string] @@ -23,13 +23,13 @@ suite "Serialization": assert result[0] == "a" assert result[1] == "b" - test "Serialize string sequence": + test "Serialization: Serialize string sequence": var input = @["a", "b"] var output = newStringStream() dump(input, output, ypsBlockOnly, ytsNone) assert output.data == "%YAML 1.2\n--- \n- a\n- b" - test "Load Table[int, string]": + test "Serialization: Load Table[int, string]": let input = newStringStream("23: dreiundzwanzig\n42: zweiundvierzig") var result: Table[int, string] @@ -41,7 +41,7 @@ suite "Serialization": assert result[23] == "dreiundzwanzig" assert result[42] == "zweiundvierzig" - test "Serialize Table[int, string]": + test "Serialization: Serialize Table[int, string]": var input = initTable[int, string]() input[23] = "dreiundzwanzig" input[42] = "zweiundvierzig" @@ -49,7 +49,7 @@ suite "Serialization": dump(input, output, ypsBlockOnly, ytsNone) assert output.data == "%YAML 1.2\n--- \n23: dreiundzwanzig\n42: zweiundvierzig" - test "Load Sequences in Sequence": + test "Serialization: Load Sequences in Sequence": let input = newStringStream(" - [1, 2, 3]\n - [4, 5]\n - [6]") var result: seq[seq[int]] @@ -62,13 +62,13 @@ suite "Serialization": assert result[1] == @[4, 5] assert result[2] == @[6] - test "Serialize Sequences in Sequence": + test "Serialization: Serialize Sequences in Sequence": let input = @[@[1, 2, 3], @[4, 5], @[6]] var output = newStringStream() dump(input, output, ypsDefault, ytsNone) assert output.data == "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]" - test "Load custom object": + test "Serialization: Load custom object": let input = newStringStream("firstname: Peter\nsurname: Pan\nage: 12") var result: Person @@ -80,13 +80,13 @@ suite "Serialization": assert result.surname == "Pan" assert result.age == 12 - test "Serialize custom object": + test "Serialization: Serialize custom object": let input = Person(firstname: "Peter", surname: "Pan", age: 12) var output = newStringStream() dump(input, output, ypsBlockOnly, ytsNone) assert output.data == "%YAML 1.2\n--- \nfirstname: Peter\nsurname: Pan\nage: 12" - test "Load sequence with explicit tags": + test "Serialization: Load sequence with explicit tags": let input = newStringStream( "--- !nim:seq(tag:yaml.org,2002:str)\n- !!str one\n- !!str two") var @@ -98,13 +98,13 @@ suite "Serialization": assert result[0] == "one" assert result[1] == "two" - test "Serialize sequence with explicit tags": + test "Serialization: Serialize sequence with explicit tags": let input = @["one", "two"] var output = newStringStream() dump(input, output, ypsBlockOnly, ytsAll) assert output.data == "%YAML 1.2\n--- !nim:seq(tag:yaml.org,2002:str) \n- !!str one\n- !!str two" - test "Load custom object with explicit root tag": + test "Serialization: Load custom object with explicit root tag": let input = newStringStream( "--- !nim:Person\nfirstname: Peter\nsurname: Pan\nage: 12") var @@ -117,7 +117,7 @@ suite "Serialization": assert result.surname == "Pan" assert result.age == 12 - test "Serialize custom object with explicit root tag": + test "Serialization: Serialize custom object with explicit root tag": let input = Person(firstname: "Peter", surname: "Pan", age: 12) var output = newStringStream() dump(input, output, ypsBlockOnly, ytsRootOnly) diff --git a/yaml.nim b/yaml.nim index 7a387b6..f946890 100644 --- a/yaml.nim +++ b/yaml.nim @@ -50,8 +50,7 @@ type ## Kinds of YAML events that may occur in an ``YamlStream``. Event kinds ## are discussed in ``YamlStreamEvent``. yamlStartDocument, yamlEndDocument, yamlStartMap, yamlEndMap, - yamlStartSequence, yamlEndSequence, yamlScalar, yamlAlias, - yamlError, yamlWarning + yamlStartSequence, yamlEndSequence, yamlScalar, yamlAlias, yamlWarning TagId* = distinct int ## \ ## A ``TagId`` identifies a tag URI, like for example @@ -62,6 +61,7 @@ type ## mapped to ``TagId`` s for efficiency reasons (you do not need to ## compare strings every time) and to be able to discover unknown tag ## URIs early in the parsing process. + AnchorId* = distinct int ## \ ## An ``AnchorId`` identifies an anchor in the current document. It ## becomes invalid as soon as the current document scope is invalidated @@ -107,7 +107,7 @@ type discard of yamlAlias: aliasTarget* : AnchorId - of yamlError, yamlWarning: + of yamlWarning: description* : string line* : int column* : int @@ -185,6 +185,47 @@ type ## - ``ypsBlockOnly``: Formats all output in block style, does not use ## flow style at all. ypsMinimal, ypsCanonical, ypsDefault, ypsJson, ypsBlockOnly + + YamlParserError* = object of Exception + ## A parser error is raised if the character stream that is parsed is + ## not a valid YAML character stream. This stream cannot and will not be + ## parsed wholly nor partially and all events that have been emitted by + ## the YamlStream the parser provides should be discarded. + ## + ## A character stream is invalid YAML if and only if at least one of the + ## following conditions apply: + ## + ## - There are invalid characters in an element whose contents is + ## restricted to a limited set of characters. For example, there are + ## characters in a tag URI which are not valid URI characters. + ## - An element has invalid indentation. This can happen for example if + ## a block list element indicated by ``"- "`` is less indented than + ## the element in the previous line, but there is no block sequence + ## list open at the same indentation level. + ## - The YAML structure is invalid. For example, an explicit block map + ## indicated by ``"? "`` and ``": "`` may not suddenly have a block + ## sequence item (``"- "``) at the same indentation level. Another + ## possible violation is closing a flow style object with the wrong + ## closing character (``}``, ``]``) or not closing it at all. + ## - A custom tag shorthand is used that has not previously been + ## declared with a ``%TAG`` directive. + ## - Multiple tags or anchors are defined for the same node. + ## - An alias is used which does not map to any anchor that has + ## previously been declared in the same document. + ## - An alias has a tag or anchor associated with it. + ## + ## Some elements in this list are vague. For a detailed description of a + ## valid YAML character stream, see the YAML specification. + line*: int ## line number (1-based) where the error was encountered + column*: int ## \ + ## column number (1-based) where the error was encountered + lineContent*: string ## \ + ## content of the line where the error was encountered. Includes a + ## second line with a marker ``^`` at the position where the error + ## was encountered, as returned by ``lexbase.getCurrentLine``. + + YamlPresenterError* = object of Exception + ## Exception that may be raised by the YAML presenter. const # failsafe schema @@ -312,7 +353,8 @@ proc newParser*(tagLib: YamlTagLibrary): YamlSequentialParser proc anchor*(parser: YamlSequentialParser, id: AnchorId): string ## Get the anchor name which an ``AnchorId`` maps to -proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream +proc parse*(parser: YamlSequentialParser, s: Stream): + YamlStream {.raises: [IOError, YamlParserError].} ## Parse a YAML character stream. ``s`` must be readable. proc constructJson*(s: YamlStream): seq[JsonNode] @@ -344,6 +386,6 @@ proc transform*(input: Stream, output: Stream, style: YamlPresentationStyle, include private.lexer include private.tagLibrary include private.events -include private.sequential +include private.parser include private.json include private.presenter \ No newline at end of file diff --git a/yaml/serialization.nim b/yaml/serialization.nim index 66b13af..a93a28e 100644 --- a/yaml/serialization.nim +++ b/yaml/serialization.nim @@ -129,7 +129,6 @@ macro make_serializable*(types: stmt): stmt = if finished(s): raise newException(ValueError, "Construction error!") while event.kind != yamlEndMap: - if event.kind == yamlError: echo event.description assert event.kind == yamlScalar assert event.scalarTag in [yTagQuestionMark, yTagString] case hash(event.scalarContent) @@ -139,7 +138,7 @@ macro make_serializable*(types: stmt): stmt = event = s() if finished(s): raise newException(ValueError, "Construction error!") - var keyCase = impl[5][1][3] + var keyCase = impl[5][1][2] assert keyCase.kind == nnkCaseStmt for field in objectFields(recList): let nameHash = hash($field.name.ident) @@ -215,7 +214,6 @@ macro make_serializable*(types: stmt): stmt = nnkIteratorDef))) serializeProc[6] = impl result.add(serializeProc) - echo result.repr proc prepend*(event: YamlStreamEvent, s: YamlStream): YamlStream = result = iterator(): YamlStreamEvent =