Implemented lookahead

private/lex.nim

@@ -43,6 +43,7 @@ type
     blockScalarIndent: int
     c: char
     tokenLineGetter: proc(lex: YamlLexer): string
+    searchColonImpl: proc(lex: YamlLexer): bool
 
   YamlLexer* = ref YamlLexerObj
 
@@ -157,6 +158,15 @@ proc nextIsPlainSafe(lex: YamlLexer, t: typedesc[StringSource],
   of flowIndicators: result = not inFlow
   else: result = true
 
+proc getPos(lex: YamlLexer, t: typedesc[BaseLexer]): int = lex.blSource.bufpos
+proc getPos(lex: YamlLexer, t: typedesc[StringSource]): int = lex.sSource.pos
+
+proc at(lex: YamlLexer, t: typedesc[BaseLexer], pos: int): char {.inline.} =
+  lex.blSource.buf[pos]
+
+proc at(lex: YamlLexer, t: typedesc[StringSource], pos: int): char {.inline.} =
+  lex.sSource.src[pos]
+
 proc mark(lex: YamlLexer, t: typedesc[BaseLexer]): int = lex.blSource.bufpos
 proc mark(lex: YamlLexer, t: typedesc[StringSource]): int = lex.sSource.pos
 
@@ -855,6 +865,78 @@ proc streamEnd(lex: YamlLexer): bool =
 proc tokenLine[T](lex: YamlLexer): string =
   result = lex.lineWithMarker(T)
 
+proc searchColon[T](lex: YamlLexer): bool =
+  var flowDepth = if lex.cur in [ltBraceOpen, ltBracketOpen]: 1 else: 0
+  let start = lex.getPos(T)
+  var
+    peek = start
+    recentAllowsAdjacent = lex.cur == ltQuotedScalar
+  result = false
+
+  proc skipPlainScalarContent(lex: YamlLexer) {.inline.} =
+    while true:
+      inc(peek)
+      case lex.at(T, peek)
+      of ']', '}', ',':
+        if flowDepth > 0 or lex.inFlow: break
+      of '#':
+        if lex.at(T, peek - 1) in space: break
+      of ':':
+        if lex.at(T, peek + 1) in spaceOrLineEnd: break
+      of lineEnd: break
+      else: discard
+
+  while peek < start + 1024:
+    case lex.at(T, peek)
+    of ':':
+      if flowDepth == 0:
+        if recentAllowsAdjacent or lex.at(T, peek + 1) in spaceOrLineEnd:
+          result = true
+          break
+        lex.skipPlainScalarContent()
+        continue
+    of '{', '[': inc(flowDepth)
+    of '}', ']':
+      dec(flowDepth)
+      if flowDepth < 0:
+        if lex.inFlow: break
+        else:
+          flowDepth = 0
+          lex.skipPlainScalarContent()
+          continue
+      recentAllowsAdjacent = true
+    of lineEnd: break
+    of '"':
+      while true:
+        inc(peek)
+        case lex.at(T, peek)
+        of lineEnd, '"': break
+        of '\\': inc(peek)
+        else: discard
+      if lex.at(T, peek) != '"': break
+      recentAllowsAdjacent = true
+    of '\'':
+      inc(peek)
+      while lex.at(T, peek) notin {'\''} + lineEnd: inc(peek)
+      if lex.at(T, peek) != '\'': break
+      recentAllowsAdjacent = true
+    of '?', ',':
+      if flowDepth == 0: break
+    of '#':
+      if lex.at(T, peek - 1) in space: break
+      lex.skipPlainScalarContent()
+      continue
+    of '&', '*', '!':
+      inc(peek)
+      while lex.at(T, peek) notin spaceOrLineEnd: inc(peek)
+      recentAllowsAdjacent = false
+      continue
+    of space: discard
+    else:
+      lex.skipPlainScalarContent()
+      continue
+    inc(peek)
+
 # interface
 
 proc init*[T](lex: YamlLexer) =
@@ -864,6 +946,7 @@ proc init*[T](lex: YamlLexer) =
   lex.insideLineImpl = insideLine[T]
   lex.insideDocImpl = insideDoc[T]
   lex.tokenLineGetter = tokenLine[T]
+  lex.searchColonImpl = searchColon[T]
 
 proc newYamlLexer*(source: Stream): YamlLexer =
   let blSource = cast[ptr BaseLexer](alloc(sizeof(BaseLexer)))
@@ -906,4 +989,7 @@ proc endBlockScalar*(lex: YamlLexer) =
   lex.nextState = lex.insideLineImpl
 
 proc getTokenLine*(lex: YamlLexer): string =
-  result = lex.tokenLineGetter(lex)
+  result = lex.tokenLineGetter(lex)
+
+proc isImplicitKeyStart*(lex: YamlLexer): bool =
+  result = lex.searchColonImpl(lex)

test/tlex.nim

@@ -90,6 +90,21 @@ proc assertEquals(input: string, expected: varargs[TokenWithValue]) =
       echo e.lineContent
       assert false
 
+proc assertLookahead(input: string, expected: bool, tokensBefore: int = 1) =
+  let lex = newYamlLexer(input)
+  var flowDepth = 0
+  for i in 0..tokensBefore:
+    lex.next()
+    case lex.cur
+    of ltBraceOpen, ltBracketOpen:
+      inc(flowDepth)
+      if flowDepth == 1: lex.setFlow(true)
+    of ltBraceClose, ltBracketClose:
+      dec(flowDepth)
+      if flowDepth == 0: lex.setFlow(false)
+    else: discard
+  doAssert lex.isImplicitKeyStart() == expected
+
 proc i(indent: int): TokenWithValue =
   TokenWithValue(kind: ltIndentation, indentation: indent)
 proc sp(v: string): TokenWithValue =
@@ -210,4 +225,35 @@ suite "Lexer":
   test "Anchors and aliases":
     assertEquals("&a foo: {&b b: *a, *b : c}", i(0), an("a"), sp("foo"), mv(),
         oo(), an("b"), sp("b"), mv(), al("a"), c(), al("b"), mv(), sp("c"),
-        oc(), se())
+        oc(), se())
+
+suite "Lookahead":
+  test "Simple Scalar":
+    assertLookahead("abcde", false)
+
+  test "Simple Mapping":
+    assertLookahead("a: b", true)
+
+  test "Colon inside plain scalar":
+    assertLookahead("abc:de", false)
+
+  test "Colon inside quoted scalar":
+    assertLookahead("'abc: de'", false)
+
+  test "Quotes inside plain scalar":
+    assertLookahead("abc\'\"de: foo", true)
+
+  test "Flow indicator inside plain scalar":
+    assertLookahead("abc}]: de", true)
+
+  test "Complex key":
+    assertLookahead("[1, 2, \"3\"]: foo", true)
+
+  test "Flow value":
+    assertLookahead("{a: b}", false)
+
+  test "In flow context":
+    assertLookahead("[ abcde]: foo", false, 2)
+
+  test "Adjacent value":
+    assertLookahead("[\"abc\":de]", true, 2)