status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fixed a GC problem; improved bench & tojson

This commit is contained in:
Felix Krause 2016-09-30 19:35:43 +02:00
parent 3942e80d9b
commit b9f569a3cf
4 changed files with 50 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
bench/jsonBench.nim
View File

@ -4,12 +4,12 @@ from nimlets_yaml import objKind
import math, strutils, stopwatch, terminal, algorithm, random, json
proc cmp(left, right: clock): int = cmp(left.nanoseconds(), right.nanoseconds())
proc cmp(left, right: clock): int = cmp(left.nanoseconds(), right.nanoseconds())
type
ObjectKind = enum
otMap, otSequence
Level = tuple
kind: ObjectKind
len: int
@ -33,10 +33,10 @@ proc genString(maxLen: int): string =
proc genJsonString(size: int, maxStringLen: int): string =
## Generates a random JSON string.
## size is in KiB, mayStringLen in characters.
randomize(size * maxStringLen)
result = "{"
let targetSize = size * 1024
var
indentation = 2
@ -44,13 +44,13 @@ proc genJsonString(size: int, maxStringLen: int): string =
curSize = 1
justOpened = true
levels.add((kind: otMap, len: 0))
while levels.len > 0:
let
objectCloseProbability =
float(levels[levels.high].len + levels.high) * 0.025
closeObject = random(1.0) <= objectCloseProbability
if (closeObject and levels.len > 1) or curSize > targetSize:
indentation -= 2
if justOpened:
@ -67,9 +67,9 @@ proc genJsonString(size: int, maxStringLen: int): string =
curSize += 1
discard levels.pop()
continue
levels[levels.high].len += 1
if justOpened:
justOpened = false
result.add("\x0A")
@ -79,7 +79,7 @@ proc genJsonString(size: int, maxStringLen: int): string =
result.add(",\x0A")
result.add(repeat(' ', indentation))
curSize += indentation + 2
case levels[levels.high].kind
of otMap:
let key = genString(maxStringLen)
@ -88,12 +88,12 @@ proc genJsonString(size: int, maxStringLen: int): string =
curSize += key.len + 2
of otSequence:
discard
let
objectValueProbability =
0.8 / float(levels.len * levels.len)
generateObjectValue = random(1.0) <= objectValueProbability
if generateObjectValue:
let objectKind = if random(2) == 0: otMap else: otSequence
case objectKind
@ -126,7 +126,7 @@ proc genJsonString(size: int, maxStringLen: int): string =
discard
else:
discard
result.add(s)
curSize += s.len
@ -138,7 +138,7 @@ var
json100k = genJsonString(100, 32)
tagLib = initCoreTagLibrary()
parser = newYamlParser(initCoreTagLibrary())
block:
multibench(cJson1k, 100):
let res = parseJson(json1k)
@ -152,24 +152,21 @@ block:
block:
multibench(cJson100k, 100):
let res = parseJson(json100k)
assert res.kind == JObject
assert res.kind == JObject
block:
multibench(cYaml1k, 100):
var events = parser.parse(json1k)
let res = constructJson(events)
let res = loadToJson(json1k)
assert res[0].kind == JObject
block:
multibench(cYaml10k, 100):
var events = parser.parse(json10k)
let res = constructJson(events)
let res = loadToJson(json10k)
assert res[0].kind == JObject
block:
multibench(cYaml100k, 100):
var events = parser.parse(json100k)
let res = constructJson(events)
let res = loadToJson(json100k)
assert res[0].kind == JObject
block:

28
bench/yamlBench.nim
View File

@ -32,16 +32,16 @@ proc genKey(): string =
let c = random(26 + 26 + 10)
if c < 26: result.add(char(c + 65))
elif c < 52: result.add(char(c + 97 - 26))
else: result.add(char(c + 48 - 52))
else: result.add(char(c + 48 - 52))
else: result = genString(31) & char(random(26) + 65)
proc genYamlString(size: int, maxStringLen: int,
style: PresentationStyle): string =
## Generates a random YAML string.
## size is in KiB, mayStringLen in characters.
randomize(size * maxStringLen * ord(style))
let targetSize = size * 1024
var
target = newStringStream()
@ -52,13 +52,13 @@ proc genYamlString(size: int, maxStringLen: int,
levels.add((kind: yMapping, len: 0))
yield startDocEvent()
yield startMapEvent()
while levels.len > 0:
let
objectCloseProbability =
float(levels[levels.high].len + levels.high) * 0.025
closeObject = random(1.0) <= objectCloseProbability
if (closeObject and levels.len > 1) or curSize > targetSize:
case levels[levels.high].kind
of yMapping: yield endMapEvent()
@ -67,19 +67,19 @@ proc genYamlString(size: int, maxStringLen: int,
curSize += 1
discard levels.pop()
continue
levels[levels.high].len += 1
if levels[levels.high].kind == yMapping:
let key = genKey()
yield scalarEvent(key)
let
objectValueProbability =
0.8 / float(levels.len * levels.len)
generateObjectValue = random(1.0) <= objectValueProbability
hasTag = random(2) == 0
var tag = yTagQuestionMark
if generateObjectValue:
let objectKind = if random(3) == 0: ySequence else: yMapping
case objectKind
@ -119,7 +119,7 @@ proc genYamlString(size: int, maxStringLen: int,
if hasTag: tag = yTagNull
else: discard
else: discard
yield scalarEvent(s, tag)
curSize += s.len
yield endDocEvent()
@ -127,7 +127,7 @@ proc genYamlString(size: int, maxStringLen: int,
present(yStream, target, initExtendedTagLibrary(),
defineOptions(style=style, outputVersion=ov1_1))
result = target.data
var
cYaml1k, cYaml10k, cYaml100k, cLibYaml1k, cLibYaml10k, cLibYaml100k,
cYaml1m, cLibYaml1m: int64
@ -153,10 +153,10 @@ block:
let res = loadDOM(yaml100k)
assert res.root.kind == yMapping
#block:
# multibench(cYaml1m, 2):
# let res = loadDOM(yaml1m)
# assert res.root.kind == yMapping
block:
multibench(cYaml1m, 2):
let res = loadDOM(yaml1m)
assert res.root.kind == yMapping
block:
multibench(cLibYaml1k, 100):

2
private/lex.nim
View File

@ -1151,7 +1151,9 @@ proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer
let sSource = safeAlloc[StringSource]()
sSource[] = StringSource(pos: startAt, lineStart: startAt, line: 1)
sSource[].src = source
GC_ref(sSource[].src)
new(result, proc(x: ref YamlLexerObj) {.nimcall.} =
GC_unref(cast[ptr StringSource](x.source)[].src)
dealloc(x.source)
)
result[] = YamlLexerObj(buf: "", source: sSource, inFlow: false,

25
yaml/tojson.nim
View File

@ -176,7 +176,8 @@ proc constructJson*(s: var YamlStream): seq[JsonNode]
else:
internalError("Unexpected node kind: " & $levels[levels.high].node.kind)
proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [YamlParserError].} =
proc loadToJson*(s: Stream): seq[JsonNode]
{.raises: [YamlParserError, YamlConstructionError, IOError].} =
## Uses `YamlParser <#YamlParser>`_ and
## `constructJson <#constructJson>`_ to construct an in-memory JSON tree
## from a YAML character stream.
@ -185,10 +186,6 @@ proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [YamlParserError].} =
events = parser.parse(s)
try:
return constructJson(events)
except YamlConstructionError:
var e = (ref YamlConstructionError)(getCurrentException())
discard events.getLastTokenContext(e.line, e.column, e.lineContent)
raise e
except YamlStreamError:
let e = getCurrentException()
if e.parent of IOError:
@ -196,6 +193,18 @@ proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [YamlParserError].} =
elif e.parent of YamlParserError:
raise (ref YamlParserError)(e.parent)
else: internalError("Unexpected exception: " & e.parent.repr)
except Exception:
# compiler bug: https://github.com/nim-lang/Nim/issues/3772
internalError("Reached code that should be unreachable")
proc loadToJson*(str: string): seq[JsonNode]
{.raises: [YamlParserError, YamlConstructionError].} =
## Uses `YamlParser <#YamlParser>`_ and
## `constructJson <#constructJson>`_ to construct an in-memory JSON tree
## from a YAML character stream.
var
parser = newYamlParser(initCoreTagLibrary())
events = parser.parse(str)
try: return constructJson(events)
except YamlStreamError:
let e = getCurrentException()
if e.parent of YamlParserError:
raise (ref YamlParserError)(e.parent)
else: internalError("Unexpected exception: " & e.parent.repr)