diff --git a/.gitignore b/.gitignore index 0a01bb6..d7182fc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,11 +3,16 @@ test/tests test/parsing test/lexing test/serializing +test/constructingJson +test/yamlTestSuite test/*.exe test/*.pdb test/*.ilk +server/server +bench/jsonBench yaml.html libyaml.dylib libyaml.so bench/json docout +yaml-dev-kit diff --git a/README.md b/README.md index 98e1aba..03eb769 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ nim server # builds the REST server used for the testing ground nim bench # runs benchmarks, requires libyaml nim clean # guess nim build # build a library +nim yamlTestSuite # execute YAML test suite (git-clones yaml-dev-kit) ``` Project is tested against current develop branch of Nim. Older Nim versions diff --git a/config.nims b/config.nims index cee793f..c15c9cd 100644 --- a/config.nims +++ b/config.nims @@ -8,6 +8,11 @@ task tests, "Run all tests": --verbosity:0 setCommand "c", "test/tests" +task yamlTestSuite, "Run YAML 1.2 test suite": + --r + --verbosity:0 + setCommand "c", "test/yamlTestSuite" + task parserTests, "Run parser tests": --r --verbosity:0 diff --git a/private/streams.nim b/private/streams.nim index 5abfe83..8abc969 100644 --- a/private/streams.nim +++ b/private/streams.nim @@ -56,6 +56,7 @@ proc finished*(s: var YamlStream): bool = raise e except Exception: let cur = getCurrentException() + echo cur.getStackTrace() var e = newException(YamlStreamError, cur.msg) e.parent = cur raise e \ No newline at end of file diff --git a/test/tmlParser.nim b/test/tmlParser.nim new file mode 100644 index 0000000..99fc0b3 --- /dev/null +++ b/test/tmlParser.nim @@ -0,0 +1,181 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +import "../yaml" +import lexbase + +type + LexerToken = enum + plusStr, minusStr, plusDoc, minusDoc, plusMap, minusMap, plusSeq, + minusSeq, eqVal, chevTag, quotContent, colonContent, noToken + + StreamPos = enum + beforeStream, inStream, afterStream + + TmlLexer = object of BaseLexer + content: string + + TmlError = object of Exception + +proc nextToken(lex: var TmlLexer): LexerToken = + while true: + case lex.buf[lex.bufpos] + of ' ', '\t': lex.bufpos.inc() + of '\r': lex.bufpos = lex.handleCR(lex.bufpos) + of '\l': lex.bufpos = lex.handleLF(lex.bufpos) + else: break + if lex.buf[lex.bufpos] == EndOfFile: return noToken + case lex.buf[lex.bufpos] + of ':', '"': + let t = if lex.buf[lex.bufpos] == ':': colonContent else: quotContent + lex.content = "" + lex.bufpos.inc() + while true: + case lex.buf[lex.bufpos] + of EndOfFile: break + of '\c': + lex.bufpos = lex.handleCR(lex.bufpos) + break + of '\l': + lex.bufpos = lex.handleLF(lex.bufpos) + break + of '\\': + lex.bufpos.inc() + case lex.buf[lex.bufpos] + of 'n': lex.content.add('\l') + of 'r': lex.content.add('\r') + of '0': lex.content.add('\0') + of '\\': lex.content.add('\\') + else: raise newException(TmlError, + "Unknown escape character: " & lex.buf[lex.bufpos]) + else: lex.content.add(lex.buf[lex.bufpos]) + lex.bufpos.inc() + result = t + of '<': + lex.content = "" + lex.bufpos.inc() + while lex.buf[lex.bufpos] != '>': + lex.content.add(lex.buf[lex.bufpos]) + lex.bufpos.inc() + result = chevTag + lex.bufpos.inc() + else: + lex.content = "" + while lex.buf[lex.bufpos] notin {' ', '\t', '\r', '\l', '\0'}: + lex.content.add(lex.buf[lex.bufpos]) + lex.bufpos.inc() + case lex.content + of "+STR": result = plusStr + of "-STR": result = minusStr + of "+DOC": result = plusDoc + of "-DOC": result = minusDoc + of "+MAP": result = plusMap + of "-MAP": result = minusMap + of "+SEQ": result = plusSeq + of "-SEQ": result = minusSeq + of "=VAL": result = eqVal + else: raise newException(TmlError, "Invalid token: " & lex.content) + +template assertInStream() {.dirty.} = + if streamPos != inStream: raise newException(TmlError, "Missing +STR") + +template assertInEvent(name: string) {.dirty.} = + if not inEvent: raise newException(TmlError, "Illegal token: " & name) + +template yieldEvent() {.dirty.} = + if inEvent: + yield curEvent + inEvent = false + +template setTag(t: TagId) {.dirty.} = + case curEvent.kind + of yamlStartSequence: curEvent.seqTag = t + of yamlStartMap: curEvent.mapTag = t + of yamlScalar: curEvent.scalarTag = t + else: discard + +template setAnchor(a: AnchorId) {.dirty.} = + case curEvent.kind + of yamlStartSequence: curEvent.seqAnchor = a + of yamlStartMap: curEvent.mapAnchor = a + of yamlScalar: curEvent.scalarAnchor = a + else: discard + +template curTag(): TagId = + var foo: TagId + case curEvent.kind + of yamlStartSequence: foo = curEvent.seqTag + of yamlStartMap: foo = curEvent.mapTag + of yamlScalar: foo = curEvent.scalarTag + else: raise newException(TmlError, $curEvent.kind & " may not have a tag") + foo + +template setCurTag(val: TagId) = + case curEvent.kind + of yamlStartSequence: curEvent.seqTag = val + of yamlStartMap: curEvent.mapTag = val + of yamlScalar: curEvent.scalarTag = val + else: raise newException(TmlError, $curEvent.kind & " may not have a tag") + +template eventStart(k: YamlStreamEventKind) {.dirty.} = + assertInStream() + yieldEvent() + reset(curEvent) + curEvent.kind = k + setTag(yTagQuestionMark) + setAnchor(yAnchorNone) + inEvent = true + +proc parseTmlStream*(input: Stream, tagLib: TagLibrary): YamlStream = + var backend = iterator(): YamlStreamEvent = + var lex: TmlLexer + lex.open(input) + var + inEvent = false + curEvent: YamlStreamEvent + streamPos = beforeStream + while lex.buf[lex.bufpos] != EndOfFile: + let token = lex.nextToken() + case token + of plusStr: + if streamPos != beforeStream: + raise newException(TmlError, "Illegal +STR") + streamPos = inStream + of minusStr: + if streamPos != inStream: + raise newException(TmlError, "Illegal -STR") + if inEvent: yield curEvent + inEvent = false + streamPos = afterStream + of plusDoc: eventStart(yamlStartDocument) + of minusDoc: eventStart(yamlEndDocument) + of plusMap: eventStart(yamlStartMap) + of minusMap: eventStart(yamlEndMap) + of plusSeq: eventStart(yamlStartSequence) + of minusSeq: eventStart(yamlEndSequence) + of eqVal: eventStart(yamlScalar) + of chevTag: + assertInEvent("tag") + if curTag() != yTagQuestionMark: + raise newException(TmlError, "Duplicate tag") + try: + setCurTag(tagLib.tags[lex.content]) + except KeyError: setCurTag(tagLib.registerUri(lex.content)) + of quotContent: + assertInEvent("scalar content") + if curTag() == yTagQuestionMark: setCurTag(yTagExclamationMark) + if curEvent.kind != yamlScalar: + raise newException(TmlError, + "scalar content in non-scalar tag") + curEvent.scalarContent = lex.content + of colonContent: + assertInEvent("scalar content") + curEvent.scalarContent = lex.content + if curEvent.kind != yamlScalar: + raise newException(TmlError, + "scalar content in non-scalar tag") + of noToken: discard + result = initYamlStream(backend) \ No newline at end of file diff --git a/test/yamlTestSuite.nim b/test/yamlTestSuite.nim new file mode 100644 index 0000000..793169b --- /dev/null +++ b/test/yamlTestSuite.nim @@ -0,0 +1,71 @@ +# NimYAML - YAML implementation in Nim +# (c) Copyright 2016 Felix Krause +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. + +import os, terminal +import tmlParser, common +import "../yaml" + +const gitCmd = + "git clone https://github.com/ingydotnet/yaml-dev-kit.git -b data" + +proc echoError(msg: string) = + styledWriteLine(stdout, fgRed, "[error] ", fgWhite, msg, resetStyle) + +proc echoInfo(msg: string) = + styledWriteLine(stdout, fgGreen, "[info] ", fgWhite, msg, resetStyle) + +removeDir("yaml-dev-kit") +if execShellCmd(gitCmd) != 0: + echoError("Could not check out yaml-dev-kit (no internet connection?)") + quit(1) + +var gotErrors = false + +try: + for kind, dirPath in walkDir("yaml-dev-kit"): + block curTest: + if kind == pcDir: + if dirPath[^4..^1] in [".git", "name", "tags"]: continue + var + tagLib = initExtendedTagLibrary() + parser = newYamlParser(tagLib) + actual = parser.parse(newFileStream(dirPath / "in.yaml")) + expected = parseTmlStream( + newFileStream(dirPath / "test.event"), tagLib) + styledWriteLine(stdout, fgGreen, "[test] ", fgWhite, + expandSymlink(dirPath / "==="), resetStyle) + while not actual.finished(): + let actualEvent = actual.next() + if expected.finished(): + echoError("Expected stream end, got " & + $actualEvent.kind) + gotErrors = true + break curTest + let expectedEvent = expected.next() + if expectedEvent != actualEvent: + printDifference(expectedEvent, actualEvent) + echoError("Actual tokens do not match expected tokens") + gotErrors = true + break curTest + if not expected.finished(): + echoError( + "Got fewer tokens than expected, first missing token: " & + $expected.next().kind) + gotErrors = true +except YamlStreamError: + let e = getCurrentException() + if e.parent of YamlParserError: + let pe = (ref YamlParserError)(e.parent) + echo "line ", pe.line, ", column ", pe.column, ": ", pe.msg + echo pe.lineContent + raise getCurrentException().parent + +if gotErrors: + echoError("There were errors while running the tests") + quit(1) +else: + echoInfo("All tests were successful") + quit(0) \ No newline at end of file