diff --git a/bench/jsonBench.nim b/bench/jsonBench.nim index ecf61e8..d187e59 100644 --- a/bench/jsonBench.nim +++ b/bench/jsonBench.nim @@ -4,12 +4,12 @@ from nimlets_yaml import objKind import math, strutils, stopwatch, terminal, algorithm, random, json -proc cmp(left, right: clock): int = cmp(left.nanoseconds(), right.nanoseconds()) +proc cmp(left, right: clock): int = cmp(left.nanoseconds(), right.nanoseconds()) type ObjectKind = enum otMap, otSequence - + Level = tuple kind: ObjectKind len: int @@ -33,10 +33,10 @@ proc genString(maxLen: int): string = proc genJsonString(size: int, maxStringLen: int): string = ## Generates a random JSON string. ## size is in KiB, mayStringLen in characters. - + randomize(size * maxStringLen) result = "{" - + let targetSize = size * 1024 var indentation = 2 @@ -44,13 +44,13 @@ proc genJsonString(size: int, maxStringLen: int): string = curSize = 1 justOpened = true levels.add((kind: otMap, len: 0)) - + while levels.len > 0: let objectCloseProbability = float(levels[levels.high].len + levels.high) * 0.025 closeObject = random(1.0) <= objectCloseProbability - + if (closeObject and levels.len > 1) or curSize > targetSize: indentation -= 2 if justOpened: @@ -67,9 +67,9 @@ proc genJsonString(size: int, maxStringLen: int): string = curSize += 1 discard levels.pop() continue - + levels[levels.high].len += 1 - + if justOpened: justOpened = false result.add("\x0A") @@ -79,7 +79,7 @@ proc genJsonString(size: int, maxStringLen: int): string = result.add(",\x0A") result.add(repeat(' ', indentation)) curSize += indentation + 2 - + case levels[levels.high].kind of otMap: let key = genString(maxStringLen) @@ -88,12 +88,12 @@ proc genJsonString(size: int, maxStringLen: int): string = curSize += key.len + 2 of otSequence: discard - + let objectValueProbability = 0.8 / float(levels.len * levels.len) generateObjectValue = random(1.0) <= objectValueProbability - + if generateObjectValue: let objectKind = if random(2) == 0: otMap else: otSequence case objectKind @@ -126,7 +126,7 @@ proc genJsonString(size: int, maxStringLen: int): string = discard else: discard - + result.add(s) curSize += s.len @@ -138,7 +138,7 @@ var json100k = genJsonString(100, 32) tagLib = initCoreTagLibrary() parser = newYamlParser(initCoreTagLibrary()) - + block: multibench(cJson1k, 100): let res = parseJson(json1k) @@ -152,24 +152,21 @@ block: block: multibench(cJson100k, 100): let res = parseJson(json100k) - assert res.kind == JObject + assert res.kind == JObject block: multibench(cYaml1k, 100): - var events = parser.parse(json1k) - let res = constructJson(events) + let res = loadToJson(json1k) assert res[0].kind == JObject block: multibench(cYaml10k, 100): - var events = parser.parse(json10k) - let res = constructJson(events) + let res = loadToJson(json10k) assert res[0].kind == JObject block: multibench(cYaml100k, 100): - var events = parser.parse(json100k) - let res = constructJson(events) + let res = loadToJson(json100k) assert res[0].kind == JObject block: diff --git a/bench/yamlBench.nim b/bench/yamlBench.nim index b5cf9d2..9cb2e6d 100644 --- a/bench/yamlBench.nim +++ b/bench/yamlBench.nim @@ -32,16 +32,16 @@ proc genKey(): string = let c = random(26 + 26 + 10) if c < 26: result.add(char(c + 65)) elif c < 52: result.add(char(c + 97 - 26)) - else: result.add(char(c + 48 - 52)) + else: result.add(char(c + 48 - 52)) else: result = genString(31) & char(random(26) + 65) proc genYamlString(size: int, maxStringLen: int, style: PresentationStyle): string = ## Generates a random YAML string. ## size is in KiB, mayStringLen in characters. - + randomize(size * maxStringLen * ord(style)) - + let targetSize = size * 1024 var target = newStringStream() @@ -52,13 +52,13 @@ proc genYamlString(size: int, maxStringLen: int, levels.add((kind: yMapping, len: 0)) yield startDocEvent() yield startMapEvent() - + while levels.len > 0: let objectCloseProbability = float(levels[levels.high].len + levels.high) * 0.025 closeObject = random(1.0) <= objectCloseProbability - + if (closeObject and levels.len > 1) or curSize > targetSize: case levels[levels.high].kind of yMapping: yield endMapEvent() @@ -67,19 +67,19 @@ proc genYamlString(size: int, maxStringLen: int, curSize += 1 discard levels.pop() continue - + levels[levels.high].len += 1 if levels[levels.high].kind == yMapping: let key = genKey() yield scalarEvent(key) - + let objectValueProbability = 0.8 / float(levels.len * levels.len) generateObjectValue = random(1.0) <= objectValueProbability hasTag = random(2) == 0 var tag = yTagQuestionMark - + if generateObjectValue: let objectKind = if random(3) == 0: ySequence else: yMapping case objectKind @@ -119,7 +119,7 @@ proc genYamlString(size: int, maxStringLen: int, if hasTag: tag = yTagNull else: discard else: discard - + yield scalarEvent(s, tag) curSize += s.len yield endDocEvent() @@ -127,7 +127,7 @@ proc genYamlString(size: int, maxStringLen: int, present(yStream, target, initExtendedTagLibrary(), defineOptions(style=style, outputVersion=ov1_1)) result = target.data - + var cYaml1k, cYaml10k, cYaml100k, cLibYaml1k, cLibYaml10k, cLibYaml100k, cYaml1m, cLibYaml1m: int64 @@ -153,10 +153,10 @@ block: let res = loadDOM(yaml100k) assert res.root.kind == yMapping -#block: -# multibench(cYaml1m, 2): -# let res = loadDOM(yaml1m) -# assert res.root.kind == yMapping +block: + multibench(cYaml1m, 2): + let res = loadDOM(yaml1m) + assert res.root.kind == yMapping block: multibench(cLibYaml1k, 100): diff --git a/private/lex.nim b/private/lex.nim index 2b67d01..cba2897 100644 --- a/private/lex.nim +++ b/private/lex.nim @@ -1151,7 +1151,9 @@ proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer let sSource = safeAlloc[StringSource]() sSource[] = StringSource(pos: startAt, lineStart: startAt, line: 1) sSource[].src = source + GC_ref(sSource[].src) new(result, proc(x: ref YamlLexerObj) {.nimcall.} = + GC_unref(cast[ptr StringSource](x.source)[].src) dealloc(x.source) ) result[] = YamlLexerObj(buf: "", source: sSource, inFlow: false, diff --git a/yaml/tojson.nim b/yaml/tojson.nim index a68f417..9c6350f 100644 --- a/yaml/tojson.nim +++ b/yaml/tojson.nim @@ -176,7 +176,8 @@ proc constructJson*(s: var YamlStream): seq[JsonNode] else: internalError("Unexpected node kind: " & $levels[levels.high].node.kind) -proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [YamlParserError].} = +proc loadToJson*(s: Stream): seq[JsonNode] + {.raises: [YamlParserError, YamlConstructionError, IOError].} = ## Uses `YamlParser <#YamlParser>`_ and ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree ## from a YAML character stream. @@ -185,10 +186,6 @@ proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [YamlParserError].} = events = parser.parse(s) try: return constructJson(events) - except YamlConstructionError: - var e = (ref YamlConstructionError)(getCurrentException()) - discard events.getLastTokenContext(e.line, e.column, e.lineContent) - raise e except YamlStreamError: let e = getCurrentException() if e.parent of IOError: @@ -196,6 +193,18 @@ proc loadToJson*(s: Stream): seq[JsonNode] {.raises: [YamlParserError].} = elif e.parent of YamlParserError: raise (ref YamlParserError)(e.parent) else: internalError("Unexpected exception: " & e.parent.repr) - except Exception: - # compiler bug: https://github.com/nim-lang/Nim/issues/3772 - internalError("Reached code that should be unreachable") + +proc loadToJson*(str: string): seq[JsonNode] + {.raises: [YamlParserError, YamlConstructionError].} = + ## Uses `YamlParser <#YamlParser>`_ and + ## `constructJson <#constructJson>`_ to construct an in-memory JSON tree + ## from a YAML character stream. + var + parser = newYamlParser(initCoreTagLibrary()) + events = parser.parse(str) + try: return constructJson(events) + except YamlStreamError: + let e = getCurrentException() + if e.parent of YamlParserError: + raise (ref YamlParserError)(e.parent) + else: internalError("Unexpected exception: " & e.parent.repr) \ No newline at end of file