import streams, unicode, lexbase import unittest type YamlTypeHint* = enum yTypeInteger, yTypeFloat, yTypeBoolean, yTypeNull, yTypeString, yTypeUnknown include "../src/private/lexer" type BasicLexerToken = tuple[kind: YamlLexerToken, content: string, typeHint: YamlTypeHint] template ensure(input: string, expected: openarray[BasicLexerToken]) = var i = 0 lex: YamlLexer lex.open(newStringStream(input)) for token in lex.tokens: if i >= expected.len: echo "received more tokens than expected (next token = ", token, ")" fail() break if token != expected[i].kind: if token == tError: echo "got lexer error: " & lex.content else: echo "wrong token kind (expected ", expected[i], ", got ", token, ")" fail() break if not isNil(expected[i].content): if lex.content != expected[i].content: echo "wrong token content (", token, ": expected \"", expected[i].content, "\", got \"", lex.content, "\")" fail() break if token == tScalarPart: if lex.typeHint != expected[i].typeHint: echo "wrong type hint (expected ", expected[i].typeHint, ", got ", lex.typeHint, ")" fail() break inc(i) if i < expected.len: echo "received less tokens than expected (first missing = ", expected[i].kind, ")" proc t(kind: YamlLexerToken, content: string, typeHint: YamlTypeHint = yTypeString): BasicLexerToken = (kind: kind, content: content, typeHint: typeHint) suite "Lexing": test "Lexing: YAML Directive": ensure("%YAML 1.2", [t(tYamlDirective, nil), t(tVersionPart, "1"), t(tVersionPart, "2"), t(tStreamEnd, nil)]) test "Lexing: TAG Directive": ensure("%TAG !t! tag:http://example.com/", [t(tTagDirective, nil), t(tTagHandle, "!t!"), t(tTagURI, "tag:http://example.com/"), t(tStreamEnd, nil)]) test "Lexing: Unknown Directive": ensure("%FOO bar baz", [t(tUnknownDirective, "%FOO"), t(tUnknownDirectiveParam, "bar"), t(tUnknownDirectiveParam, "baz"), t(tStreamEnd, nil)]) test "Lexing: Comments after Directives": ensure("%YAML 1.2 # version\n# at line start\n # indented\n%FOO", [t(tYamlDirective, nil), t(tVersionPart, "1"), t(tVersionPart, "2"), t(tComment, " version"), t(tComment, " at line start"), t(tComment, " indented"), t(tUnknownDirective, "%FOO"), t(tStreamEnd, nil)]) test "Lexing: Directives End": ensure("---", [t(tDirectivesEnd, nil), t(tStreamEnd, nil)]) test "Lexing: Document End": ensure("...", [t(tLineStart, nil), t(tDocumentEnd, nil), t(tStreamEnd, nil)]) test "Lexing: Directive after Document End": ensure("content\n...\n%YAML 1.2", [t(tLineStart, ""), t(tScalarPart, "content"), t(tLineStart, ""), t(tDocumentEnd, nil), t(tYamlDirective, nil), t(tVersionPart, "1"), t(tVersionPart, "2"), t(tStreamEnd, nil)]) test "Lexing: Plain Scalar (alphanumeric)": ensure("abA03rel4", [t(tLineStart, ""), t(tScalarPart, "abA03rel4"), t(tStreamEnd, nil)]) test "Lexing: Plain Scalar (with spaces)": ensure("test content", [t(tLineStart, ""), t(tScalarPart, "test content"), t(tStreamEnd, nil)]) test "Lexing: Plain Scalar (with special chars)": ensure(":test ?content -with #special !chars", [t(tLineStart, nil), t(tScalarPart, ":test ?content -with #special !chars"), t(tStreamEnd, nil)]) test "Lexing: Plain Scalar (starting with %)": ensure("---\n%test", [t(tDirectivesEnd, nil), t(tLineStart, ""), t(tScalarPart, "%test"), t(tStreamEnd, nil)]) test "Lexing: Single Quoted Scalar": ensure("'? test - content! '", [t(tLineStart, ""), t(tScalar, "? test - content! "), t(tStreamEnd, nil)]) test "Lexing: Single Quoted Scalar (escaped single quote inside)": ensure("'test '' content'", [t(tLineStart, ""), t(tScalar, "test ' content"), t(tStreamEnd, nil)]) test "Lexing: Doubly Quoted Scalar": ensure("\"test content\"", [t(tLineStart, ""), t(tScalar, "test content"), t(tStreamEnd, nil)]) test "Lexing: Doubly Quoted Scalar (escaping)": ensure(""""\t\\\0\""""", [t(tLineStart, ""), t(tScalar, "\t\\\0\""), t(tStreamEnd, nil)]) test "Lexing: Doubly Quoted Scalar (unicode escaping)": ensure(""""\x42\u4243\U00424344"""", [t(tLineStart, ""), t(tScalar, "\x42" & toUTF8(cast[Rune](0x4243)) & toUTF8(cast[Rune](0x424344))), t(tStreamEnd, nil)]) test "Lexing: Block Array": ensure(""" - a - b""", [t(tLineStart, ""), t(tDash, nil), t(tScalarPart, "a"), t(tLineStart, ""), t(tDash, nil), t(tScalarPart, "b"), t(tStreamEnd, nil)]) test "Lexing: Block Map with Implicit Keys": ensure(""" foo: bar herp: derp""", [t(tLineStart, ""), t(tScalarPart, "foo"), t(tColon, nil), t(tScalarPart, "bar"), t(tLineStart, ""), t(tScalarPart, "herp"), t(tColon, nil), t(tScalarPart, "derp"), t(tStreamEnd, nil)]) test "Lexing: Block Map with Explicit Keys": ensure(""" ? foo : bar""", [t(tLineStart, ""), t(tQuestionmark, nil), t(tScalarPart, "foo"), t(tLineStart, ""), t(tColon, nil), t(tScalarPart, "bar"), t(tStreamEnd, nil)]) test "Lexing: Indentation": ensure(""" foo: bar: - baz - biz herp: derp""", [t(tLineStart, ""), t(tScalarPart, "foo"), t(tColon, nil), t(tLineStart, " "), t(tScalarPart, "bar"), t(tColon, nil), t(tLineStart, " "), t(tDash, nil), t(tScalarPart, "baz"), t(tLineStart, " "), t(tDash, nil), t(tScalarPart, "biz"), t(tLineStart, " "), t(tScalarPart, "herp"), t(tColon, nil), t(tScalarPart, "derp"), t(tStreamEnd, nil)]) test "Lexing: Anchor": ensure("foo: &bar", [t(tLineStart, ""), t(tScalarPart, "foo"), t(tColon, nil), t(tAnchor, "bar"), t(tStreamEnd, nil)]) test "Lexing: Alias": ensure("foo: *bar", [t(tLineStart, ""), t(tScalarPart, "foo"), t(tColon, nil), t(tAlias, "bar"), t(tStreamEnd, nil)]) test "Lexing: Tag handle": ensure("!t!str tagged", [t(tLineStart, ""), t(tTagHandle, "!t!"), t(tTagSuffix, "str"), t(tScalarPart, "tagged"), t(tStreamEnd, nil)]) test "Lexing: Verbatim tag handle": ensure("! tagged", [t(tLineStart, ""), t(tVerbatimTag, "tag:http://example.com/str"), t(tScalarPart, "tagged"), t(tStreamEnd, nil)]) test "Lexing: Type hints": ensure("false\nnull\nstring\n-13\n42.25\n-4e+3\n5.42e78", [t(tLineStart, ""), t(tScalarPart, "false", yTypeBoolean), t(tLineStart, ""), t(tScalarPart, "null", yTypeNull), t(tLineStart, ""), t(tScalarPart, "string", yTypeString), t(tLineStart, ""), t(tScalarPart, "-13", yTypeInteger), t(tLineStart, ""), t(tScalarPart, "42.25", yTypeFloat), t(tLineStart, ""), t(tScalarPart, "-4e+3", yTypeFloat), t(tLineStart, ""), t(tScalarPart, "5.42e78", yTypeFloat), t(tStreamEnd, nil)])