diff --git a/src/yaml/private/lexer.nim b/src/yaml/private/lexer.nim index 285454c..057025d 100644 --- a/src/yaml/private/lexer.nim +++ b/src/yaml/private/lexer.nim @@ -370,6 +370,7 @@ iterator tokens*(my: var YamlLexer): YamlLexerEvent = yieldError("Unsupported escape sequence: \\" & c) if expectedEscapeLength == 0: state = ylDoublyQuotedScalar else: + let digitPosition = expectedEscapeLength - escapeLength - 1 case c of EndOFFile: yieldError("Unterminated escape sequence") @@ -377,15 +378,16 @@ iterator tokens*(my: var YamlLexer): YamlLexerEvent = continue of '0' .. '9': unicodeChar = unicodechar or - (cast[int](c) - 0x30) shl ((4 - escapeLength) * 8) + (cast[int](c) - 0x30) shl (digitPosition * 4) of 'A' .. 'F': unicodeChar = unicodechar or - (cast[int](c) - 0x37) shl ((4 - escapeLength) * 8) + (cast[int](c) - 0x37) shl (digitPosition * 4) of 'a' .. 'f': unicodeChar = unicodechar or - (cast[int](c) - 0x57) shl ((4 - escapeLength) * 8) + (cast[int](c) - 0x57) shl (digitPosition * 4) else: - yieldError("unsupported char in unicode escape sequence: " & c) + yieldError("unsupported char in unicode escape sequence: " & + c) escapeLength = 0 state = ylDoublyQuotedScalar continue diff --git a/test/lexing.nim b/test/lexing.nim index 542ee8e..e23c83e 100644 --- a/test/lexing.nim +++ b/test/lexing.nim @@ -1,5 +1,5 @@ import "../src/yaml/private/lexer" -import streams +import streams, unicode import unittest @@ -69,6 +69,16 @@ suite "Lexing": ensure("...", [t(yamlDocumentEnd, nil), t(yamlStreamEnd, nil)]) + test "Directive after document end": + ensure("content\n...\n%YAML 1.2", + [t(yamlLineStart, nil), + t(yamlScalar, "content"), + t(yamlDocumentEnd, nil), + t(yamlYamlDirective, nil), + t(yamlMajorVersion, "1"), + t(yamlMinorVersion, "2"), + t(yamlStreamEnd, nil)]) + test "Plain Scalar (alphanumeric)": ensure("abA03rel4", [t(yamlLineStart, nil), t(yamlScalar, "abA03rel4"), @@ -99,4 +109,21 @@ suite "Lexing": test "Single Quoted Scalar (escaped single quote inside)": ensure("'test '' content'", [t(yamlLineStart, nil), t(yamlScalar, "test ' content"), - t(yamlStreamEnd, nil)]) \ No newline at end of file + t(yamlStreamEnd, nil)]) + + test "Doubly Quoted Scalar": + ensure("\"test content\"", [t(yamlLineStart, nil), + t(yamlScalar, "test content"), + t(yamlStreamEnd, nil)]) + + test "Doubly Quoted Scalar (escaping)": + ensure(""""\t\\\0\""""", [t(yamlLineStart, nil), + t(yamlScalar, "\t\\\0\""), + t(yamlStreamEnd, nil)]) + + test "Doubly Quoted Scalar (unicode escaping)": + ensure(""""\x42\u4243\U00424344"""", + [t(yamlLineStart, nil), + t(yamlScalar, "\x42" & toUTF8(cast[Rune](0x4243)) & + toUTF8(cast[Rune](0x424344))), + t(yamlStreamEnd, nil)]) \ No newline at end of file