Improved tests, benchs, shallowCopies

This commit is contained in:
Felix Krause 2016-09-14 18:31:09 +02:00
parent 2655d4205f
commit 2bb7f45354
10 changed files with 88 additions and 83 deletions

View File

@ -129,10 +129,12 @@ proc genYamlString(size: int, maxStringLen: int,
result = target.data
var
cYaml1k, cYaml10k, cYaml100k, cLibYaml1k, cLibYaml10k, cLibYaml100k: int64
cYaml1k, cYaml10k, cYaml100k, cLibYaml1k, cLibYaml10k, cLibYaml100k,
cYaml1m, cLibYaml1m: int64
yaml1k = genYamlString(1, 32, psDefault)
yaml10k = genYamlString(10, 32, psDefault)
yaml100k = genYamlString(100, 32, psDefault)
yaml1m = genYamlString(1000, 32, psDefault)
tagLib = initExtendedTagLibrary()
parser = newYamlParser(tagLib)
@ -151,6 +153,11 @@ block:
let res = loadDOM(yaml100k)
assert res.root.kind == yMapping
block:
multibench(cYaml1m, 2):
let res = loadDOM(yaml1m)
assert res.root.kind == yMapping
block:
multibench(cLibYaml1k, 100):
let res = nimlets_yaml.load(yaml1k)
@ -166,6 +173,11 @@ block:
let res = nimlets_yaml.load(yaml100k)
assert res[0].objKind == nimlets_yaml.YamlObjKind.Map
block:
multibench(cLibYaml1m, 2):
let res = nimlets_yaml.load(yaml1m)
assert res[0].objKind == nimlets_yaml.YamlObjKind.Map
proc writeResult(caption: string, num: int64) =
styledWriteLine(stdout, resetStyle, caption, fgGreen, $num, resetStyle, "μs")
@ -184,3 +196,7 @@ setForegroundColor(fgWhite)
writeStyled "100k input\n----------\n"
writeResult "NimYAML: ", cYaml100k div 1000
writeResult "LibYAML: ", cLibYaml100k div 1000
setForegroundColor(fgWhite)
writeStyled "1m input\n---------\n"
writeResult "NimYAML: ", cYaml1m div 1000
writeResult "LibYAML: ", cLibYaml1m div 1000

View File

@ -20,7 +20,8 @@ proc initYamlDoc*(root: YamlNode): YamlDocument = result.root = root
proc composeNode(s: var YamlStream, tagLib: TagLibrary,
c: ConstructionContext):
YamlNode {.raises: [YamlStreamError, YamlConstructionError].} =
let start = s.next()
var start: YamlStreamEvent
shallowCopy(start, s.next())
new(result)
try:
case start.kind
@ -64,7 +65,8 @@ proc composeNode(s: var YamlStream, tagLib: TagLibrary,
proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument
{.raises: [YamlStreamError, YamlConstructionError].} =
var context = newConstructionContext()
var n = s.next()
var n: YamlStreamEvent
shallowCopy(n, s.next())
yAssert n.kind == yamlStartDoc
result.root = composeNode(s, tagLib, context)
n = s.next()

View File

@ -1148,7 +1148,7 @@ proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer
try:
let sSource = safeAlloc[StringSource]()
sSource[] = StringSource(pos: startAt, lineStart: startAt, line: 1)
shallowCopy(sSource[].src, source)
sSource[].src = source
new(result, proc(x: ref YamlLexerObj) {.nimcall.} =
dealloc(x.source)
)

View File

@ -75,10 +75,11 @@ proc emptyScalar(c: ParserContext): YamlStreamEvent {.raises: [], inline.} =
c.tag = yTagQuestionMark
c.anchor = yAnchorNone
proc currentScalar(c: ParserContext): YamlStreamEvent {.raises: [], inline.} =
result = YamlStreamEvent(kind: yamlScalar, scalarTag: c.tag,
proc currentScalar(c: ParserContext, e: var YamlStreamEvent)
{.raises: [], inline.} =
e = YamlStreamEvent(kind: yamlScalar, scalarTag: c.tag,
scalarAnchor: c.anchor)
shallowCopy(result.scalarContent, c.lex.buf)
shallowCopy(e.scalarContent, c.lex.buf)
c.lex.buf = cast[string not nil](newStringOfCap(256))
c.tag = yTagQuestionMark
c.anchor = yAnchorNone
@ -563,13 +564,13 @@ parserState blockObjectStart:
parserState scalarEnd:
if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark
e = c.currentScalar()
c.currentScalar(e)
result = true
state = objectEnd
stored = blockAfterObject
parserState plainScalarEnd:
e = c.currentScalar()
c.currentScalar(e)
result = true
state = objectEnd
stored = blockAfterObject
@ -807,7 +808,7 @@ parserState flow:
of ltQuotedScalar:
if c.handleFlowItemStart(e): return true
if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark
e = c.currentScalar()
c.currentScalar(e)
result = true
state = objectEnd
stored = flowAfterObject
@ -837,7 +838,7 @@ parserState flow:
of ltScalarPart:
if c.handleFlowItemStart(e): return true
c.handleFlowPlainScalar()
e = c.currentScalar()
c.currentScalar(e)
result = true
state = objectEnd
stored = flowAfterObject

View File

@ -69,7 +69,7 @@ proc next*(s: YamlStream): YamlStreamEvent =
proc peek*(s: YamlStream): YamlStreamEvent =
if not s.peeked:
s.cached = s.next()
shallowCopy(s.cached, s.next())
s.peeked = true
shallowCopy(result, s.cached)

View File

@ -29,13 +29,13 @@ proc ensureEqual(yamlIn, jsonIn: string) =
raise e
suite "Constructing JSON":
test "Constructing JSON: Simple Sequence":
test "Simple Sequence":
ensureEqual("- 1\n- 2\n- 3", "[1, 2, 3]")
test "Constructing JSON: Simple Map":
test "Simple Map":
ensureEqual("a: b\nc: d", """{"a": "b", "c": "d"}""")
test "Constructing JSON: Complex Structure":
test "Complex Structure":
ensureEqual("""
%YAML 1.2
---

View File

@ -8,18 +8,18 @@ import "../yaml"
import unittest, common
suite "DOM":
test "DOM: Composing simple Scalar":
test "Composing simple Scalar":
let
input = newStringStream("scalar")
result = loadDOM(input)
assert result.root.kind == yScalar
assert result.root.content == "scalar"
assert result.root.tag == "?"
test "DOM: Serializing simple Scalar":
test "Serializing simple Scalar":
let input = initYamlDoc(newYamlNode("scalar"))
var result = serialize(input, initExtendedTagLibrary())
ensure(result, startDocEvent(), scalarEvent("scalar"), endDocEvent())
test "DOM: Composing sequence":
test "Composing sequence":
let
input = newStringStream("- !!str a\n- !!bool no")
result = loadDOM(input)
@ -32,7 +32,7 @@ suite "DOM":
assert result.root.children[1].kind == yScalar
assert result.root.children[1].tag == "tag:yaml.org,2002:bool"
assert result.root.children[1].content == "no"
test "DOM: Serializing sequence":
test "Serializing sequence":
let input = initYamlDoc(newYamlNode([
newYamlNode("a", "tag:yaml.org,2002:str"),
newYamlNode("no", "tag:yaml.org,2002:bool")]))
@ -40,7 +40,7 @@ suite "DOM":
ensure(result, startDocEvent(), startSeqEvent(),
scalarEvent("a", yTagString), scalarEvent("no", yTagBoolean),
endSeqEvent(), endDocEvent())
test "DOM: Composing mapping":
test "Composing mapping":
let
input = newStringStream("--- !!map\n!foo bar: [a, b]")
result = loadDOM(input)
@ -52,7 +52,7 @@ suite "DOM":
assert result.root.pairs[0].key.content == "bar"
assert result.root.pairs[0].value.kind == ySequence
assert result.root.pairs[0].value.children.len == 2
test "DOM: Serializing mapping":
test "Serializing mapping":
let input = initYamlDoc(newYamlNode([
(key: newYamlNode("bar"), value: newYamlNode([newYamlNode("a"),
newYamlNode("b")]))]))
@ -60,7 +60,7 @@ suite "DOM":
ensure(result, startDocEvent(), startMapEvent(), scalarEvent("bar"),
startSeqEvent(), scalarEvent("a"), scalarEvent("b"),
endSeqEvent(), endMapEvent(), endDocEvent())
test "DOM: Composing with anchors":
test "Composing with anchors":
let
input = newStringStream("- &a foo\n- &b bar\n- *a\n- *b")
result = loadDOM(input)
@ -72,7 +72,7 @@ suite "DOM":
assert result.root.children[1].content == "bar"
assert result.root.children[0] == result.root.children[2]
assert result.root.children[1] == result.root.children[3]
test "DOM: Serializing with anchors":
test "Serializing with anchors":
let
a = newYamlNode("a")
b = newYamlNode("b")
@ -83,7 +83,7 @@ suite "DOM":
scalarEvent("b", anchor=1.AnchorId), scalarEvent("c"),
aliasEvent(0.AnchorId), aliasEvent(1.AnchorId), endSeqEvent(),
endDocEvent())
test "DOM: Serializing with all anchors":
test "Serializing with all anchors":
let
a = newYamlNode("a")
input = initYamlDoc(newYamlNode([a, newYamlNode("b"), a]))

View File

@ -77,7 +77,7 @@ proc newNode(v: string): ref Node =
let blockOnly = defineOptions(style=psBlockOnly)
suite "Serialization":
test "Serialization: Load integer without fixed length":
test "Load integer without fixed length":
var input = newStringStream("-4247")
var result: int
load(input, result)
@ -89,7 +89,7 @@ suite "Serialization":
except: gotException = true
assert gotException, "Expected exception, got none."
test "Serialization: Dump integer without fixed length":
test "Dump integer without fixed length":
var input = -4247
var output = newStringStream()
dump(input, output, tsNone, asTidy, blockOnly)
@ -102,19 +102,19 @@ suite "Serialization":
except: gotException = true
assert gotException, "Expected exception, got none."
test "Serialization: Load nil string":
test "Load nil string":
let input = newStringStream("!nim:nil:string \"\"")
var result: string
load(input, result)
assert isNil(result)
test "Serialization: Dump nil string":
test "Dump nil string":
let input: string = nil
var output = newStringStream()
dump(input, output, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n!nim:nil:string \"\"", output.data
test "Serialization: Load string sequence":
test "Load string sequence":
let input = newStringStream(" - a\n - b")
var result: seq[string]
load(input, result)
@ -122,25 +122,25 @@ suite "Serialization":
assert result[0] == "a"
assert result[1] == "b"
test "Serialization: Dump string sequence":
test "Dump string sequence":
var input = @["a", "b"]
var output = newStringStream()
dump(input, output, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output.data
test "Serialization: Load nil seq":
test "Load nil seq":
let input = newStringStream("!nim:nil:seq \"\"")
var result: seq[int]
load(input, result)
assert isNil(result)
test "Serialization: Dump nil seq":
test "Dump nil seq":
let input: seq[int] = nil
var output = newStringStream()
dump(input, output, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n!nim:nil:seq \"\"", output.data
test "Serialization: Load char set":
test "Load char set":
let input = newStringStream("- a\n- b")
var result: set[char]
load(input, result)
@ -148,13 +148,13 @@ suite "Serialization":
assert 'a' in result
assert 'b' in result
test "Serialization: Dump char set":
test "Dump char set":
var input = {'a', 'b'}
var output = newStringStream()
dump(input, output, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n- a\n- b", output.data
test "Serialization: Load array":
test "Load array":
let input = newStringStream("- 23\n- 42\n- 47")
var result: array[0..2, int32]
load(input, result)
@ -162,13 +162,13 @@ suite "Serialization":
assert result[1] == 42
assert result[2] == 47
test "Serialization: Dump array":
test "Dump array":
let input = [23'i32, 42'i32, 47'i32]
var output = newStringStream()
dump(input, output, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n- 23\n- 42\n- 47", output.data
test "Serialization: Load Table[int, string]":
test "Load Table[int, string]":
let input = newStringStream("23: dreiundzwanzig\n42: zweiundvierzig")
var result: Table[int32, string]
load(input, result)
@ -176,7 +176,7 @@ suite "Serialization":
assert result[23] == "dreiundzwanzig"
assert result[42] == "zweiundvierzig"
test "Serialization: Dump Table[int, string]":
test "Dump Table[int, string]":
var input = initTable[int32, string]()
input[23] = "dreiundzwanzig"
input[42] = "zweiundvierzig"
@ -185,7 +185,7 @@ suite "Serialization":
assertStringEqual("%YAML 1.2\n--- \n23: dreiundzwanzig\n42: zweiundvierzig",
output.data)
test "Serialization: Load OrderedTable[tuple[int32, int32], string]":
test "Load OrderedTable[tuple[int32, int32], string]":
let input = newStringStream("- {a: 23, b: 42}: drzw\n- {a: 13, b: 47}: drsi")
var result: OrderedTable[tuple[a, b: int32], string]
load(input, result)
@ -201,7 +201,7 @@ suite "Serialization":
else: assert false
i.inc()
test "Serialization: Dump OrderedTable[tuple[int32, int32], string]":
test "Dump OrderedTable[tuple[int32, int32], string]":
var input = initOrderedTable[tuple[a, b: int32], string]()
input.add((a: 23'i32, b: 42'i32), "dreiundzwanzigzweiundvierzig")
input.add((a: 13'i32, b: 47'i32), "dreizehnsiebenundvierzig")
@ -220,7 +220,7 @@ suite "Serialization":
b: 47
: dreizehnsiebenundvierzig""", output.data)
test "Serialization: Load Sequences in Sequence":
test "Load Sequences in Sequence":
let input = newStringStream(" - [1, 2, 3]\n - [4, 5]\n - [6]")
var result: seq[seq[int32]]
load(input, result)
@ -229,14 +229,14 @@ suite "Serialization":
assert result[1] == @[4.int32, 5.int32]
assert result[2] == @[6.int32]
test "Serialization: Dump Sequences in Sequence":
test "Dump Sequences in Sequence":
let input = @[@[1.int32, 2.int32, 3.int32], @[4.int32, 5.int32], @[6.int32]]
var output = newStringStream()
dump(input, output, tsNone)
assertStringEqual "%YAML 1.2\n--- \n- [1, 2, 3]\n- [4, 5]\n- [6]",
output.data
test "Serialization: Load Enum":
test "Load Enum":
let input = newStringStream("!nim:system:seq(tl)\n- !tl tlRed\n- tlGreen\n- tlYellow")
var result: seq[TrafficLight]
load(input, result)
@ -245,14 +245,14 @@ suite "Serialization":
assert result[1] == tlGreen
assert result[2] == tlYellow
test "Serialization: Dump Enum":
test "Dump Enum":
let input = @[tlRed, tlGreen, tlYellow]
var output = newStringStream()
dump(input, output, tsNone, asTidy, blockOnly)
assertStringEqual "%YAML 1.2\n--- \n- tlRed\n- tlGreen\n- tlYellow",
output.data
test "Serialization: Load Tuple":
test "Load Tuple":
let input = newStringStream("str: value\ni: 42\nb: true")
var result: MyTuple
load(input, result)
@ -260,13 +260,13 @@ suite "Serialization":
assert result.i == 42
assert result.b == true
test "Serialization: Dump Tuple":
test "Dump Tuple":
let input = (str: "value", i: 42.int32, b: true)
var output = newStringStream()
dump(input, output, tsNone)
assertStringEqual "%YAML 1.2\n--- \nstr: value\ni: 42\nb: y", output.data
test "Serialization: Load custom object":
test "Load custom object":
let input = newStringStream("firstnamechar: P\nsurname: Pan\nage: 12")
var result: Person
load(input, result)
@ -274,7 +274,7 @@ suite "Serialization":
assert result.surname == "Pan"
assert result.age == 12
test "Serialization: Dump custom object":
test "Dump custom object":
let input = Person(firstnamechar: 'P', surname: "Pan", age: 12)
var output = newStringStream()
dump(input, output, tsNone, asTidy, blockOnly)
@ -289,14 +289,14 @@ suite "Serialization":
assert result[0] == "one"
assert result[1] == "two"
test "Serialization: Dump sequence with explicit tags":
test "Dump sequence with explicit tags":
let input = @["one", "two"]
var output = newStringStream()
dump(input, output, tsAll, asTidy, blockOnly)
assertStringEqual("%YAML 1.2\n--- !nim:system:seq(" &
"tag:yaml.org,2002:str) \n- !!str one\n- !!str two", output.data)
test "Serialization: Load custom object with explicit root tag":
test "Load custom object with explicit root tag":
let input = newStringStream(
"--- !nim:custom:Person\nfirstnamechar: P\nsurname: Pan\nage: 12")
var result: Person
@ -305,7 +305,7 @@ suite "Serialization":
assert result.surname == "Pan"
assert result.age == 12
test "Serialization: Dump custom object with explicit root tag":
test "Dump custom object with explicit root tag":
let input = Person(firstnamechar: 'P', surname: "Pan", age: 12)
var output = newStringStream()
dump(input, output, tsRootOnly, asTidy, blockOnly)
@ -313,7 +313,7 @@ suite "Serialization":
"--- !nim:custom:Person \nfirstnamechar: P\nsurname: Pan\nage: 12",
output.data)
test "Serialization: Load custom variant object":
test "Load custom variant object":
let input = newStringStream(
"---\n- - name: Bastet\n - kind: akCat\n - purringIntensity: 7\n" &
"- - name: Anubis\n - kind: akDog\n - barkometer: 13")
@ -327,7 +327,7 @@ suite "Serialization":
assert result[1].kind == akDog
assert result[1].barkometer == 13
test "Serialization: Dump custom variant object":
test "Dump custom variant object":
let input = @[Animal(name: "Bastet", kind: akCat, purringIntensity: 7),
Animal(name: "Anubis", kind: akDog, barkometer: 13)]
var output = newStringStream()
@ -349,7 +349,7 @@ suite "Serialization":
-
barkometer: 13""", output.data
test "Serialization: Dump cyclic data structure":
test "Dump cyclic data structure":
var
a = newNode("a")
b = newNode("b")
@ -368,7 +368,7 @@ next:
value: c
next: *a""", output.data
test "Serialization: Load cyclic data structure":
test "Load cyclic data structure":
let input = newStringStream("""%YAML 1.2
--- !nim:system:seq(example.net:Node)
- &a
@ -397,7 +397,7 @@ next:
assert(result[1].next == result[2])
assert(result[2].next == result[0])
test "Serialization: Load nil values":
test "Load nil values":
let input = newStringStream("- ~\n- !!str ~")
var result: seq[ref string]
try: load(input, result)
@ -411,7 +411,7 @@ next:
assert(result[0] == nil)
assert(result[1][] == "~")
test "Serialization: Dump nil values":
test "Dump nil values":
var input = newSeq[ref string]()
input.add(nil)
input.add(new string)
@ -422,7 +422,7 @@ next:
"%YAML 1.2\n--- !nim:system:seq(tag:yaml.org,2002:str) \n- !!null ~\n- !!str ~",
output.data)
test "Serialization: Custom constructObject":
test "Custom constructObject":
let input = newStringStream("- 1\n- !test:BetterInt 2")
var result: seq[BetterInt]
load(input, result)
@ -430,7 +430,7 @@ next:
assert(result[0] == 2.BetterInt)
assert(result[1] == 3.BetterInt)
test "Serialization: Custom representObject":
test "Custom representObject":
let input = @[1.BetterInt, 9998887.BetterInt, 98312.BetterInt]
var output = newStringStream()
dump(input, output, tsAll, asTidy, blockOnly)

View File

@ -4,4 +4,4 @@
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
import constructingJson, serializing, dom
import tlex, constructingJson, serializing, dom

View File

@ -5,7 +5,7 @@ import unittest, strutils
const tokensWithValue =
{ltScalarPart, ltQuotedScalar, ltYamlVersion, ltTagShorthand, ltTagUri,
ltUnknownDirective, ltUnknownDirectiveParams, ltLiteralTag, ltAnchor,
ltAlias}
ltAlias, ltBlockScalar}
type
TokenWithValue = object
@ -14,9 +14,6 @@ type
value: string
of ltIndentation:
indentation: int
of ltBlockScalarHeader:
folded: bool
chomp: ChompType
of ltTagHandle:
handle, suffix: string
else: discard
@ -28,8 +25,6 @@ proc actualRepr(lex: YamlLexer, t: LexerToken): string =
result.add("(" & escape(lex.buf) & ")")
of ltIndentation:
result.add("(" & $lex.indentation & ")")
of ltBlockScalarHeader:
result.add("(" & $lex.folded & ", " & $lex.chomp & ")")
else: discard
proc assertEquals(input: string, expected: varargs[TokenWithValue]) =
@ -58,14 +53,6 @@ proc assertEquals(input: string, expected: varargs[TokenWithValue]) =
if lex.indentation <= blockScalarEnd:
lex.endBlockScalar()
blockScalarEnd = -1
of ltBlockScalarHeader:
doAssert lex.folded == expectedToken.folded,
"Wrong folded indicator at #" & $i & ": Expected " &
$expectedToken.folded & ", got " & $lex.folded
doAssert lex.chomp == expectedToken.chomp,
"Wrong chomp indicator at #" & $i & ": Expected " &
$expectedToken.chomp & ", got " & $lex.chomp
blockScalarEnd = lex.indentation
of ltBraceOpen, ltBracketOpen:
inc(flowDepth)
if flowDepth == 1: lex.setFlow(true)
@ -129,8 +116,9 @@ proc tu(v: string): TokenWithValue =
TokenWithValue(kind: ltTagUri, value: v)
proc dirE(): TokenWithValue = TokenWithValue(kind: ltDirectivesEnd)
proc docE(): TokenWithValue = TokenWithValue(kind: ltDocumentEnd)
proc bs(folded: bool, chomp: ChompType): TokenWithValue =
TokenWithValue(kind: ltBlockScalarHeader, folded: folded, chomp: chomp)
proc bsh(): TokenWithValue = TokenWithValue(kind: ltBlockScalarHeader)
proc bs(v: string): TokenWithValue =
TokenWithValue(kind: ltBlockScalar, value: v)
proc el(): TokenWithValue = TokenWithValue(kind: ltEmptyLine)
proc ao(): TokenWithValue = TokenWithValue(kind: ltBracketOpen)
proc ac(): TokenWithValue = TokenWithValue(kind: ltBracketClose)
@ -196,14 +184,12 @@ suite "Lexer":
docE(), du("UNKNOWN"), dp("warbl"), se())
test "Block scalar":
assertEquals("|\l a\l\l b\l # comment", i(0), bs(false, ctClip), i(2),
sp("a"), el(), i(2), sp("b"), i(1), se())
assertEquals("|\l a\l\l b\l # comment", i(0), bsh(), bs("a\l\lb\l"), se())
test "Block Scalars":
assertEquals("one : >2-\l foo\l bar\ltwo: |+\l bar\l baz", i(0),
sp("one"), mv(), bs(true, ctStrip), i(3), sp(" foo"), i(2), sp("bar"),
i(0), sp("two"), mv(), bs(false, ctKeep), i(1), sp("bar"), i(2),
sp(" baz"), se())
sp("one"), mv(), bsh(), bs(" foo\lbar"), i(0), sp("two"), mv(), bsh(),
bs("bar\l baz"), se())
test "Flow indicators":
assertEquals("bla]: {c: d, [e]: f}", i(0), sp("bla]"), mv(), oo(), sp("c"),