status-im/NimYAML
2
0
Fork 0
mirror of https://github.com/status-im/NimYAML.git synced 2025-01-27 11:34:56 +00:00
Code Issues Projects Releases Wiki Activity

Fixed unicode escaping; added tests

Browse Source
This commit is contained in:
Felix Krause 2015-11-29 21:01:22 +01:00
parent f793247a41
commit 71456c6bed
2 changed files with 35 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/yaml/private/lexer.nim
View File

@ -370,6 +370,7 @@ iterator tokens*(my: var YamlLexer): YamlLexerEvent =
yieldError("Unsupported escape sequence: \\" & c) yieldError("Unsupported escape sequence: \\" & c)
if expectedEscapeLength == 0: state = ylDoublyQuotedScalar if expectedEscapeLength == 0: state = ylDoublyQuotedScalar
else: else:
let digitPosition = expectedEscapeLength - escapeLength - 1
case c case c
of EndOFFile: of EndOFFile:
yieldError("Unterminated escape sequence") yieldError("Unterminated escape sequence")
@ -377,15 +378,16 @@ iterator tokens*(my: var YamlLexer): YamlLexerEvent =
continue continue
of '0' .. '9': of '0' .. '9':
unicodeChar = unicodechar or unicodeChar = unicodechar or
(cast[int](c) - 0x30) shl ((4 - escapeLength) * 8) (cast[int](c) - 0x30) shl (digitPosition * 4)
of 'A' .. 'F': of 'A' .. 'F':
unicodeChar = unicodechar or unicodeChar = unicodechar or
(cast[int](c) - 0x37) shl ((4 - escapeLength) * 8) (cast[int](c) - 0x37) shl (digitPosition * 4)
of 'a' .. 'f': of 'a' .. 'f':
unicodeChar = unicodechar or unicodeChar = unicodechar or
(cast[int](c) - 0x57) shl ((4 - escapeLength) * 8) (cast[int](c) - 0x57) shl (digitPosition * 4)
else: else:
yieldError("unsupported char in unicode escape sequence: " & c) yieldError("unsupported char in unicode escape sequence: " &
c)
escapeLength = 0 escapeLength = 0
state = ylDoublyQuotedScalar state = ylDoublyQuotedScalar
continue continue

31
test/lexing.nim
View File

@ -1,5 +1,5 @@
import "../src/yaml/private/lexer" import "../src/yaml/private/lexer"
import streams import streams, unicode
import unittest import unittest
@ -69,6 +69,16 @@ suite "Lexing":
ensure("...", [t(yamlDocumentEnd, nil), ensure("...", [t(yamlDocumentEnd, nil),
t(yamlStreamEnd, nil)]) t(yamlStreamEnd, nil)])
test "Directive after document end":
ensure("content\n...\n%YAML 1.2",
[t(yamlLineStart, nil),
t(yamlScalar, "content"),
t(yamlDocumentEnd, nil),
t(yamlYamlDirective, nil),
t(yamlMajorVersion, "1"),
t(yamlMinorVersion, "2"),
t(yamlStreamEnd, nil)])
test "Plain Scalar (alphanumeric)": test "Plain Scalar (alphanumeric)":
ensure("abA03rel4", [t(yamlLineStart, nil), ensure("abA03rel4", [t(yamlLineStart, nil),
t(yamlScalar, "abA03rel4"), t(yamlScalar, "abA03rel4"),
@ -99,4 +109,21 @@ suite "Lexing":
test "Single Quoted Scalar (escaped single quote inside)": test "Single Quoted Scalar (escaped single quote inside)":
ensure("'test '' content'", [t(yamlLineStart, nil), ensure("'test '' content'", [t(yamlLineStart, nil),
t(yamlScalar, "test ' content"), t(yamlScalar, "test ' content"),
t(yamlStreamEnd, nil)]) t(yamlStreamEnd, nil)])
test "Doubly Quoted Scalar":
ensure("\"test content\"", [t(yamlLineStart, nil),
t(yamlScalar, "test content"),
t(yamlStreamEnd, nil)])
test "Doubly Quoted Scalar (escaping)":
ensure(""""\t\\\0\""""", [t(yamlLineStart, nil),
t(yamlScalar, "\t\\\0\""),
t(yamlStreamEnd, nil)])
test "Doubly Quoted Scalar (unicode escaping)":
ensure(""""\x42\u4243\U00424344"""",
[t(yamlLineStart, nil),
t(yamlScalar, "\x42" & toUTF8(cast[Rune](0x4243)) &
toUTF8(cast[Rune](0x424344))),
t(yamlStreamEnd, nil)])