Fixed bench and remaining problems

bench/jsonBench.nim

@@ -156,25 +156,19 @@ block:
 
 block:
     multibench(cYaml1k, 100):
-        var
+        var events = parser.parse(json1k)
-            s = newStringStream(json1k)
-            events = parser.parse(s)
         let res = constructJson(events)
         assert res[0].kind == JObject
 
 block:
     multibench(cYaml10k, 100):
-        var
+        var events = parser.parse(json10k)
-            s = newStringStream(json10k)
-            events = parser.parse(s)
         let res = constructJson(events)
         assert res[0].kind == JObject
 
 block:
     multibench(cYaml100k, 100):
-        var
+        var events = parser.parse(json100k)
-            s = newStringStream(json100k)
-            events = parser.parse(s)
         let res = constructJson(events)
         assert res[0].kind == JObject
 

bench/yamlBench.nim

@@ -138,20 +138,17 @@ var
 
 block:
     multibench(cYaml1k, 100):
-        var s = newStringStream(yaml1k)
+        let res = loadDOM(yaml1k)
-        let res = loadDOM(s)
         assert res.root.kind == yMapping
 
 block:
     multibench(cYaml10k, 100):
-        var s = newStringStream(yaml10k)
+        let res = loadDOM(yaml10k)
-        let res = loadDOM(s)
         assert res.root.kind == yMapping
 
 block:
     multibench(cYaml100k, 100):
-        var s = newStringStream(yaml100k)
+        let res = loadDOM(yaml100k)
-        let res = loadDOM(s)
         assert res.root.kind == yMapping
 
 block:

config.nims

@@ -28,7 +28,6 @@ task documentation, "Generate documentation":
   setCommand "nop"
 
 task bench, "Benchmarking":
-  --d:release
   --r
   --w:off
   --hints:off

private/dom.nim

@@ -70,7 +70,7 @@ proc compose*(s: var YamlStream, tagLib: TagLibrary): YamlDocument
   n = s.next()
   yAssert n.kind == yamlEndDoc
 
-proc loadDOM*(s: Stream): YamlDocument
+proc loadDOM*(s: Stream | string): YamlDocument
     {.raises: [IOError, YamlParserError, YamlConstructionError].} =
   var
     tagLib = initExtendedTagLibrary()

private/lex.nim

@@ -804,6 +804,19 @@ proc blockScalarHeader[T](lex: YamlLexer): bool =
   lex.cur = ltBlockScalarHeader
   result = true
 
+proc blockScalarAfterLineStart[T](lex: YamlLexer,
+    recentWasMoreIndented: var bool): bool =
+  if lex.indentation < lex.blockScalarIndent:
+    lex.nextState = indentationAfterBlockScalar[T]
+    return false
+
+  if lex.folded and not recentWasMoreIndented: lex.consumeNewlines()
+  else:
+    recentWasMoreIndented = false
+    lex.buf.add(repeat('\l', lex.newlines))
+    lex.newlines = 0
+  result = true
+
 proc blockScalarLineStart[T](lex: YamlLexer, recentWasMoreIndented: var bool):
     bool =
   while true:
@@ -854,17 +867,7 @@ proc blockScalarLineStart[T](lex: YamlLexer, recentWasMoreIndented: var bool):
         return true
       else: break
     else: break
-
+  result = blockScalarAfterLineStart[T](lex, recentWasMoreIndented)
-  if lex.indentation < lex.blockScalarIndent:
-    lex.nextState = indentationAfterBlockScalar[T]
-    return false
-
-  if lex.folded and not recentWasMoreIndented: lex.consumeNewlines()
-  else:
-    recentWasMoreIndented = false
-    lex.buf.add(repeat('\l', lex.newlines))
-    lex.newlines = 0
-  result = true
 
 proc blockScalar[T](lex: YamlLexer): bool =
   debug("lex: blockScalar")
@@ -895,13 +898,18 @@ proc blockScalar[T](lex: YamlLexer): bool =
           break
     else:
       lex.blockScalarIndent += lex.indentation
-    if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer
+    if lex.c notin {'.', '-'} or lex.indentation == 0:
+      if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer
+    else:
+      if not blockScalarAfterLineStart[T](lex, recentWasMoreIndented):
+        break outer
     while true:
       while lex.c notin lineEnd:
         lex.buf.add(lex.c)
         lex.advance(T)
       if not blockScalarLineStart[T](lex, recentWasMoreIndented): break outer
   
+  debug("lex: leaving block scalar at indentation " & $lex.indentation)
   case lex.chomp
   of ctStrip: discard
   of ctClip:
@@ -911,7 +919,6 @@ proc blockScalar[T](lex: YamlLexer): bool =
   lex.lineStartState = insideDoc[T]
   lex.cur = ltBlockScalar
   result = true
-  echo "exiting block scalar with indentation=" & $lex.indentation
 
 proc indentationAfterBlockScalar[T](lex: YamlLexer): bool =
   if lex.indentation == 0:
@@ -1119,25 +1126,37 @@ proc init*[T](lex: YamlLexer) =
   lex.tokenLineGetter = tokenLine[T]
   lex.searchColonImpl = searchColon[T]
 
-proc newYamlLexer*(source: Stream): YamlLexer =
+proc safeAlloc[T](): ptr T =
-  let blSource = cast[ptr BaseLexer](alloc0(sizeof(BaseLexer)))
+  try: result = cast[ptr T](alloc0(sizeof(T)))
-  blSource[].open(source)
+  except:
-  new(result, proc(x: ref YamlLexerObj) {.nimcall.} =
+    discard # TODO
-      dealloc(x.source)
+
-  )
+proc newYamlLexer*(source: Stream): YamlLexer {.raises: [].} =
-  result[] = YamlLexerObj(source: blSource, inFlow: false, buf: "",
+  try:
-      c: blSource[].buf[blSource[].bufpos], newlines: 0, folded: true)
+    let blSource = safeAlloc[BaseLexer]()
+    blSource[].open(source)
+    new(result, proc(x: ref YamlLexerObj) {.nimcall.} =
+        dealloc(x.source)
+    )
+    result[] = YamlLexerObj(source: blSource, inFlow: false, buf: "",
+        c: blSource[].buf[blSource[].bufpos], newlines: 0, folded: true)
+  except:
+    discard # TODO
   init[BaseLexer](result)
 
-proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer =
+proc newYamlLexer*(source: string, startAt: int = 0): YamlLexer
-  let sSource = cast[ptr StringSource](alloc(sizeof(StringSource)))
+    {.raises: [].}=
-  sSource[] =
+  try:
-      StringSource(src: source, pos: startAt, lineStart: startAt, line: 1)
+    let sSource = safeAlloc[StringSource]()
-  new(result, proc(x: ref YamlLexerObj) {.nimcall.} =
+    sSource[] = StringSource(pos: startAt, lineStart: startAt, line: 1)
-      dealloc(x.source)
+    shallowCopy(sSource[].src, source)
-  )
+    new(result, proc(x: ref YamlLexerObj) {.nimcall.} =
-  result[] = YamlLexerObj(buf: "", source: sSource, inFlow: false,
+        dealloc(x.source)
-      c: sSource.src[startAt], newlines: 0, folded: true)
+    )
+    result[] = YamlLexerObj(buf: "", source: sSource, inFlow: false,
+        c: sSource.src[startAt], newlines: 0, folded: true)
+  except:
+    discard # TODO
   init[StringSource](result)
 
 proc next*(lex: YamlLexer) =

private/parse.nim

@@ -54,6 +54,7 @@ proc illegalToken(c: ParserContext, expected: string = ""):
   if expected.len > 0: msg.add(" (expected " & expected & ")")
   msg.add(": " & $c.lex.cur)
   result = c.generateError(msg)
+  echo result.line, ", ", result.column
 
 proc callCallback(c: ParserContext, msg: string) {.raises: [YamlParserError].} =
   try:

private/serialization.nim

@@ -758,7 +758,7 @@ proc construct*[T](s: var YamlStream, target: var T) =
     ex.parent = getCurrentException()
     raise ex
 
-proc load*[K](input: Stream, target: var K) =
+proc load*[K](input: Stream | string, target: var K) =
   var
     parser = newYamlParser(serializationTagLibrary)
     events = parser.parse(input)
@@ -804,3 +804,11 @@ proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly,
   try: present(events, target, serializationTagLibrary, options)
   except YamlStreamError:
     internalError("Unexpected exception: " & getCurrentException().repr)
+
+proc dump*[K](value: K, tagStyle: TagStyle = tsRootOnly,
+              anchorStyle: AnchorStyle = asTidy,
+              options: PresentationOptions = defaultPresentationOptions):
+    string =
+  var s = newStringStream()
+  dump(value, s, tagStyle, anchorStyle, options)
+  shallowCopy(result, s.data)

yaml.nim

@@ -523,6 +523,9 @@ proc parse*(p: YamlParser, s: Stream): YamlStream {.raises: [YamlParserError].}
   ## Parse the given stream as YAML character stream.
   ## The only Exception that can be raised comes from opening the Stream.
 
+proc parse*(p: YamlParser, str: string): YamlStream
+    {.raises: [YamlParserError].}
+
 proc defineOptions*(style: PresentationStyle = psDefault,
                     indentationStep: int = 2,
                     newlines: NewLineStyle = nlOSDefault,
@@ -611,7 +614,7 @@ proc construct*[T](s: var YamlStream, target: var T)
     {.raises: [YamlStreamError].}
   ## Constructs a Nim value from a YAML stream.
 
-proc load*[K](input: Stream, target: var K)
+proc load*[K](input: Stream | string, target: var K)
     {.raises: [YamlConstructionError, IOError, YamlParserError].}
   ## Loads a Nim value from a YAML character stream.
 
@@ -627,6 +630,13 @@ proc dump*[K](value: K, target: Stream, tagStyle: TagStyle = tsRootOnly,
                YamlStreamError].}
   ## Dump a Nim value as YAML character stream.
 
+proc dump*[K](value: K, tagStyle: TagStyle = tsRootOnly,
+              anchorStyle: AnchorStyle = asTidy,
+              options: PresentationOptions = defaultPresentationOptions):
+    string {.raises: [YamlPresenterJsonError, YamlPresenterOutputError,
+                      YamlStreamError].}
+  ## Dump a Nim value as YAML into a string
+
 var
   serializationTagLibrary* = initSerializationTagLibrary() ## \
     ## contains all local tags that are used for type serialization. Does