From 4670450a27b3de8639bbd6c80900080f40abb4fd Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Wed, 14 Sep 2016 14:35:41 +0200 Subject: [PATCH] Fixed bench and remaining problems --- bench/jsonBench.nim | 12 ++---- bench/yamlBench.nim | 9 ++--- config.nims | 1 - private/dom.nim | 2 +- private/lex.nim | 79 ++++++++++++++++++++++++--------------- private/parse.nim | 1 + private/serialization.nim | 10 ++++- yaml.nim | 12 +++++- 8 files changed, 77 insertions(+), 49 deletions(-) diff --git a/bench/jsonBench.nim b/bench/jsonBench.nim index 8fedd26..05de3b9 100644 --- a/bench/jsonBench.nim +++ b/bench/jsonBench.nim @@ -156,25 +156,19 @@ block: block: multibench(cYaml1k, 100): - var - s = newStringStream(json1k) - events = parser.parse(s) + var events = parser.parse(json1k) let res = constructJson(events) assert res[0].kind == JObject block: multibench(cYaml10k, 100): - var - s = newStringStream(json10k) - events = parser.parse(s) + var events = parser.parse(json10k) let res = constructJson(events) assert res[0].kind == JObject block: multibench(cYaml100k, 100): - var - s = newStringStream(json100k) - events = parser.parse(s) + var events = parser.parse(json100k) let res = constructJson(events) assert res[0].kind == JObject diff --git a/bench/yamlBench.nim b/bench/yamlBench.nim index 2624a70..60db16d 100644 --- a/bench/yamlBench.nim +++ b/bench/yamlBench.nim @@ -138,20 +138,17 @@ var block: multibench(cYaml1k, 100): - var s = newStringStream(yaml1k) - let res = loadDOM(s) + let res = loadDOM(yaml1k) assert res.root.kind == yMapping block: multibench(cYaml10k, 100): - var s = newStringStream(yaml10k) - let res = loadDOM(s) + let res = loadDOM(yaml10k) assert res.root.kind == yMapping block: multibench(cYaml100k, 100): - var s = newStringStream(yaml100k) - let res = loadDOM(s) + let res = loadDOM(yaml100k) assert res.root.kind == yMapping block: diff --git a/config.nims b/config.nims index 1021e14..b6bb6c2 100644 --- a/config.nims +++ b/config.nims @@ -28,7 +28,6 @@ task documentation, "Generate documentation": setCommand "nop" task bench, "Benchmarking": - --d:release --r --w:off --hints:off diff --git a/private/dom.nim b/private/dom.nim index 27cc9f5..91608aa 100644 --- a/private/dom.nim +++ b/private/dom.nim @@ -70,7 +70,7 @@ proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument n = s.next() yAssert n.kind == yamlEndDoc -proc loadDOM*(s: Stream): YamlDocument +proc loadDOM*(s: Stream | string): YamlDocument {.raises: [IOError, YamlParserError, YamlConstructionError].} = var tagLib = initExtendedTagLibrary() diff --git a/private/lex.nim b/private/lex.nim index 10d8e94..cc13dc6 100644 --- a/private/lex.nim +++ b/private/lex.nim @@ -804,6 +804,19 @@ proc blockScalarHeader[T](lex: YamlLexer): bool = lex.cur = ltBlockScalarHeader result = true +proc blockScalarAfterLineStart[T](lex: YamlLexer, + recentWasMoreIndented: var bool): bool = + if lex.indentation < lex.blockScalarIndent: + lex.nextState = indentationAfterBlockScalar[T] + return false + + if lex.folded and not recentWasMoreIndented: lex.consumeNewlines() + else: + recentWasMoreIndented = false + lex.buf.add(repeat('\l', lex.newlines)) + lex.newlines = 0 + result = true + proc blockScalarLineStart[T](lex: YamlLexer, recentWasMoreIndented: var bool): bool = while true: @@ -854,17 +867,7 @@ proc blockScalarLineStart[T](lex: YamlLexer, recentWasMoreIndented: var bool): return true else: break else: break - - if lex.indentation < lex.blockScalarIndent: - lex.nextState = indentationAfterBlockScalar[T] - return false - - if lex.folded and not recentWasMoreIndented: lex.consumeNewlines() - else: - recentWasMoreIndented = false - lex.buf.add(repeat('\l', lex.newlines)) - lex.newlines = 0 - result = true + result = blockScalarAfterLineStart[T](lex, recentWasMoreIndented) proc blockScalar[T](lex: YamlLexer): bool = debug("lex: blockScalar") @@ -895,13 +898,18 @@ proc blockScalar[T](lex: YamlLexer): bool = break else: lex.blockScalarIndent += lex.indentation - if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer + if lex.c notin {'.', '-'} or lex.indentation == 0: + if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer + else: + if not blockScalarAfterLineStart[T](lex, recentWasMoreIndented): + break outer while true: while lex.c notin lineEnd: lex.buf.add(lex.c) lex.advance(T) if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer + debug("lex: leaving block scalar at indentation " & $lex.indentation) case lex.chomp of ctStrip: discard of ctClip: @@ -911,7 +919,6 @@ proc blockScalar[T](lex: YamlLexer): bool = lex.lineStartState = insideDoc[T] lex.cur = ltBlockScalar result = true - echo "exiting block scalar with indentation=" & $lex.indentation proc indentationAfterBlockScalar[T](lex: YamlLexer): bool = if lex.indentation == 0: @@ -1119,25 +1126,37 @@ proc init*[T](lex: YamlLexer) = lex.tokenLineGetter = tokenLine[T] lex.searchColonImpl = searchColon[T] -proc newYamlLexer*(source: Stream): YamlLexer = - let blSource = cast[ptr BaseLexer](alloc0(sizeof(BaseLexer))) - blSource[].open(source) - new(result, proc(x: ref YamlLexerObj) {.nimcall.} = - dealloc(x.source) - ) - result[] = YamlLexerObj(source: blSource, inFlow: false, buf: "", - c: blSource[].buf[blSource[].bufpos], newlines: 0, folded: true) +proc safeAlloc[T](): ptr T = + try: result = cast[ptr T](alloc0(sizeof(T))) + except: + discard # TODO + +proc newYamlLexer*(source: Stream): YamlLexer {.raises: [].} = + try: + let blSource = safeAlloc[BaseLexer]() + blSource[].open(source) + new(result, proc(x: ref YamlLexerObj) {.nimcall.} = + dealloc(x.source) + ) + result[] = YamlLexerObj(source: blSource, inFlow: false, buf: "", + c: blSource[].buf[blSource[].bufpos], newlines: 0, folded: true) + except: + discard # TODO init[BaseLexer](result) -proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer = - let sSource = cast[ptr StringSource](alloc(sizeof(StringSource))) - sSource[] = - StringSource(src: source, pos: startAt, lineStart: startAt, line: 1) - new(result, proc(x: ref YamlLexerObj) {.nimcall.} = - dealloc(x.source) - ) - result[] = YamlLexerObj(buf: "", source: sSource, inFlow: false, - c: sSource.src[startAt], newlines: 0, folded: true) +proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer + {.raises: [].}= + try: + let sSource = safeAlloc[StringSource]() + sSource[] = StringSource(pos: startAt, lineStart: startAt, line: 1) + shallowCopy(sSource[].src, source) + new(result, proc(x: ref YamlLexerObj) {.nimcall.} = + dealloc(x.source) + ) + result[] = YamlLexerObj(buf: "", source: sSource, inFlow: false, + c: sSource.src[startAt], newlines: 0, folded: true) + except: + discard # TODO init[StringSource](result) proc next*(lex: YamlLexer) = diff --git a/private/parse.nim b/private/parse.nim index a9c4862..217c1e0 100644 --- a/private/parse.nim +++ b/private/parse.nim @@ -54,6 +54,7 @@ proc illegalToken(c: ParserContext, expected: string = ""): if expected.len > 0: msg.add(" (expected " & expected & ")") msg.add(": " & $c.lex.cur) result = c.generateError(msg) + echo result.line, ", ", result.column proc callCallback(c: ParserContext, msg: string) {.raises: [YamlParserError].} = try: diff --git a/private/serialization.nim b/private/serialization.nim index 15db621..4b1aac3 100644 --- a/private/serialization.nim +++ b/private/serialization.nim @@ -758,7 +758,7 @@ proc construct*[T](s: var YamlStream, target: var T) = ex.parent = getCurrentException() raise ex -proc load*[K](input: Stream, target: var K) = +proc load*[K](input: Stream | string, target: var K) = var parser = newYamlParser(serializationTagLibrary) events = parser.parse(input) @@ -804,3 +804,11 @@ proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly, try: present(events, target, serializationTagLibrary, options) except YamlStreamError: internalError("Unexpected exception: " & getCurrentException().repr) + +proc dump*[K](value: K, tagStyle: TagStyle = tsRootOnly, + anchorStyle: AnchorStyle = asTidy, + options: PresentationOptions = defaultPresentationOptions): + string = + var s = newStringStream() + dump(value, s, tagStyle, anchorStyle, options) + shallowCopy(result, s.data) \ No newline at end of file diff --git a/yaml.nim b/yaml.nim index eb9bc1d..7f97bda 100644 --- a/yaml.nim +++ b/yaml.nim @@ -523,6 +523,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream {.raises: [YamlParserError].} ## Parse the given stream as YAML character stream. ## The only Exception that can be raised comes from opening the Stream. +proc parse*(p: YamlParser, str: string): YamlStream + {.raises: [YamlParserError].} + proc defineOptions*(style: PresentationStyle = psDefault, indentationStep: int = 2, newlines: NewLineStyle = nlOSDefault, @@ -611,7 +614,7 @@ proc construct*[T](s: var YamlStream, target: var T) {.raises: [YamlStreamError].} ## Constructs a Nim value from a YAML stream. -proc load*[K](input: Stream, target: var K) +proc load*[K](input: Stream | string, target: var K) {.raises: [YamlConstructionError, IOError, YamlParserError].} ## Loads a Nim value from a YAML character stream. @@ -627,6 +630,13 @@ proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly, YamlStreamError].} ## Dump a Nim value as YAML character stream. +proc dump*[K](value: K, tagStyle: TagStyle = tsRootOnly, + anchorStyle: AnchorStyle = asTidy, + options: PresentationOptions = defaultPresentationOptions): + string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError, + YamlStreamError].} + ## Dump a Nim value as YAML into a string + var serializationTagLibrary* = initSerializationTagLibrary() ## \ ## contains all local tags that are used for type serialization. Does