Implemented block scalars

src/yaml/private/lexer.nim

@@ -25,8 +25,7 @@ type
         yamlOpeningBracket, yamlClosingBrace, yamlClosingBracket, yamlPipe,
         yamlGreater,
         # block scalar header
-        yamlLiteralScalar, yamlFoldedScalar,
+        yamlBlockIndentationIndicator, yamlPlus,
-        yamlBlockIndentationIndicator, yamlBlockChompingIndicator,
         # scalar content
         yamlScalar, yamlScalarPart,
         # tags
@@ -802,9 +801,10 @@ iterator tokens*(my: var YamlLexer): YamlLexerToken {.closure.} =
             of '0' .. '9':
                 my.content = "" & c
                 yieldToken(yamlBlockIndentationIndicator)
-            of '+', '-':
+            of '+':
-                my.content = "" & c
+                yieldToken(yamlPlus)
-                yieldToken(yamlBlockChompingIndicator)
+            of '-':
+                yieldToken(yamlDash)
             of '\r', '\x0A', EndOfFile:
                 blockScalarIndentation = lastIndentationLength
                 state = ylLineEnd

src/yaml/sequential.nim

@@ -29,18 +29,24 @@ type
     YamlParserState = enum
         ylInitial, ylSkipDirective, ylBlockLineStart, ylBlockAfterTag,
         ylBlockAfterAnchor, ylBlockAfterScalar, ylBlockAfterColon,
-        ylBlockMultilineScalar, ylBlockLineEnd, ylFlow, ylFlowAfterObject,
+        ylBlockMultilineScalar, ylBlockLineEnd, ylBlockScalarHeader,
-        ylExpectingDocumentEnd
+        ylBlockScalar, ylFlow, ylFlowAfterObject, ylExpectingDocumentEnd
     
     DocumentLevelMode = enum
         mBlockSequenceItem, mFlowSequenceItem, mExplicitBlockMapKey,
         mExplicitBlockMapValue, mImplicitBlockMapKey, mImplicitBlockMapValue,
-        mFlowMapKey, mFlowMapValue, mPlainScalar, mScalar, mUnknown
+        mFlowMapKey, mFlowMapValue, mScalar, mUnknown
     
     DocumentLevel = object
         mode: DocumentLevelMode
         indicatorColumn: int
         indentationColumn: int
+    
+    LineStrippingMode = enum
+        lsStrip, lsClip, lsKeep
+    
+    BlockScalarStyle = enum
+        bsLiteral, bsFolded
 
 proc `==`*(left: YamlParserEvent, right: YamlParserEvent): bool =
     if left.kind != right.kind:
@@ -156,6 +162,12 @@ iterator events*(input: Stream): YamlParserEvent {.closure.} =
         level = DocumentLevel(mode: mUnknown, indicatorColumn: -1,
                               indentationColumn: -1)
         
+        # block scalar state
+        lineStrip: LineStrippingMode
+        blockScalar: BlockScalarStyle
+        blockScalarIndentation: int
+        blockScalarTrailing: string = nil
+        
         # cached values
         tag: string = nil
         anchor: string = nil
@@ -239,6 +251,13 @@ iterator events*(input: Stream): YamlParserEvent {.closure.} =
             of yamlColon:
                 handleBlockIndicator(mExplicitBlockMapKey,
                                      mExplicitBlockMapValue, yamlError)
+            of yamlPipe, yamlGreater:
+                blockScalar = if token == yamlPipe: bsLiteral else: bsFolded
+                blockScalarIndentation = -1
+                lineStrip = lsClip
+                state = ylBlockScalarHeader
+                scalarCache = ""
+                level.mode = mScalar
             of yamlTagHandle:
                 let handle = lex.content
                 if tagShorthands.hasKey(handle):
@@ -431,6 +450,13 @@ iterator events*(input: Stream): YamlParserEvent {.closure.} =
             of yamlOpeningBracket, yamlOpeningBrace:
                 state = ylFlow
                 continue
+            of yamlPipe, yamlGreater:
+                blockScalar = if token == yamlPipe: bsLiteral else: bsFolded
+                blockScalarIndentation = -1
+                lineStrip = lsClip
+                state = ylBlockScalarHeader
+                scalarCache = ""
+                level.mode = mScalar
             else:
                 yieldError("Unexpected token (expected scalar or line end): " &
                            $token)
@@ -445,6 +471,84 @@ iterator events*(input: Stream): YamlParserEvent {.closure.} =
                 break
             else:
                 yieldError("Unexpected token (expected line end):" & $token)
+        of ylBlockScalarHeader:
+            case token
+            of yamlPlus:
+                if lineStrip != lsClip:
+                    yieldError("Multiple chomping indicators!")
+                else:
+                    lineStrip = lsKeep
+            of yamlDash:
+                if lineStrip != lsClip:
+                    yieldError("Multiple chomping indicators!")
+                else:
+                    lineStrip = lsStrip
+            of yamlBlockIndentationIndicator:
+                if blockScalarIndentation != -1:
+                    yieldError("Multiple indentation indicators!")
+                else:
+                    blockScalarIndentation = parseInt(lex.content)
+            of yamlLineStart:
+                blockScalarTrailing = ""
+                state = ylBlockScalar
+            else:
+                yieldError("Unexpected token: " & $token)
+        of ylBlockScalar:
+            case token
+            of yamlLineStart:
+                if level.indentationColumn == -1:
+                    discard
+                else:
+                    case blockScalar
+                    of bsLiteral:
+                        blockScalarTrailing &= "\x0A"
+                    of bsFolded:
+                        case blockScalarTrailing.len
+                        of 0:
+                            blockScalarTrailing = " "
+                        of 1:
+                            blockScalarTrailing = "\x0A"
+                        else:
+                            discard
+                    
+                    if lex.content.len > level.indentationColumn:
+                        if blockScalar == bsFolded:
+                            if blockScalarTrailing == " ":
+                                blockScalarTrailing = "\x0A"
+                        scalarCache &= blockScalarTrailing &
+                                lex.content[level.indentationColumn..^1]
+                        blockScalarTrailing = ""
+                            
+            of yamlScalarPart:
+                if ancestry.high > 0:
+                    if ancestry[ancestry.high].indicatorColumn >= lex.column or
+                       ancestry[ancestry.high].indicatorColumn == -1 and
+                       ancestry[ancestry.high].indentationColumn >= lex.column:
+                            # todo: trailing chomping?
+                            closeLevel(level)
+                            state = ylBlockLineStart
+                            continue
+                if level.indentationColumn == -1:
+                    level.indentationColumn = lex.column
+                else:
+                    scalarCache &= blockScalarTrailing
+                    blockScalarTrailing = ""
+                scalarCache &= lex.content
+            else:
+                case lineStrip
+                of lsStrip:
+                    discard
+                of lsClip:
+                    scalarCache &= "\x0A"
+                of lsKeep:
+                    scalarCache &= blockScalarTrailing
+                closeLevel(level)
+                if ancestry.len == 0:
+                    state = ylExpectingDocumentEnd
+                else:
+                    level = ancestry.pop()
+                    state = ylBlockLineStart
+                continue            
         of ylFlow:
             case token
             of yamlLineStart:

test/parsing.nim

@@ -57,8 +57,20 @@ proc printDifference(expected, actual: YamlParserEvent) =
                 echo "[scalar] expected anchor " & expected.scalarAnchor &
                      ", got " & actual.scalarAnchor
             elif expected.scalarContent != actual.scalarContent:
-                echo "[scalar] expected content \"" & expected.scalarContent &
+                let msg = "[scalar] expected content \"" &
-                     "\", got \"" & actual.scalarContent & "\""
+                        expected.scalarContent & "\", got \"" &
+                        actual.scalarContent & "\" "
+                for i in 0..expected.scalarContent.high:
+                    if i >= actual.scalarContent.high:
+                        echo msg, "(expected more chars, first char missing: ",
+                             cast[int](expected.scalarContent[i]), ")"
+                        break
+                    elif expected.scalarContent[i] != actual.scalarContent[i]:
+                        echo msg, "(first different char at pos ", i,
+                                ": expected ",
+                                cast[int](expected.scalarContent[i]), ", got ",
+                                cast[int](actual.scalarContent[i]), ")"
+                        break
             else:
                 echo "[scalar] Unknown difference"
         else:
@@ -132,4 +144,18 @@ suite "Parsing":
         ensure("a\nb  \n  c\nd", startDoc(), scalar("a b c d"), endDoc())
     test "Parsing: Multiline scalar (in map)":
         ensure("a: b\n c\nd:\n e\n  f", startDoc(), startMap(), scalar("a"),
-               scalar("b c"), scalar("d"), scalar("e f"), endMap(), endDoc())
+               scalar("b c"), scalar("d"), scalar("e f"), endMap(), endDoc())
+    test "Parsing: Block scalar (literal)":
+        ensure("a: |\x0A ab\x0A \x0A cd\x0A ef\x0A \x0A", startDoc(),
+               startMap(), scalar("a"), scalar("ab\x0A\x0Acd\x0Aef\x0A"),
+               endMap(), endDoc())
+    test "Parsing: Block scalar (folded)":
+        ensure("a: >\x0A ab\x0A cd\x0A \x0Aef\x0A\x0A\x0Agh\x0A", startDoc(),
+               startMap(), scalar("a"), scalar("ab cd\x0Aef\x0Agh\x0A"),
+               endMap(), endDoc())
+    test "Parsing: Block scalar (keep)":
+        ensure("a: |+\x0A ab\x0A \x0A  \x0A", startDoc(), startMap(),
+               scalar("a"), scalar("ab\x0A\x0A \x0A"), endMap(), endDoc())
+    test "Parsing: Block scalar (strip)":
+        ensure("a: |-\x0A ab\x0A \x0A \x0A", startDoc(), startMap(),
+               scalar("a"), scalar("ab"), endMap(), endDoc())