2015-11-27 22:42:11 +00:00
|
|
|
import lexbase, unicode, streams
|
|
|
|
|
|
|
|
type
|
|
|
|
Encoding* = enum
|
|
|
|
Unsupported, ## Unsupported encoding
|
|
|
|
UTF8, ## UTF-8
|
|
|
|
UTF16LE, ## UTF-16 Little Endian
|
|
|
|
UTF16BE, ## UTF-16 Big Endian
|
|
|
|
UTF32LE, ## UTF-32 Little Endian
|
|
|
|
UTF32BE ## UTF-32 Big Endian
|
|
|
|
|
|
|
|
YamlLexerEventKind* = enum
|
2015-11-29 14:42:41 +00:00
|
|
|
# separating tokens
|
|
|
|
yamlDirectivesEnd, yamlDocumentEnd, yamlStreamEnd,
|
|
|
|
# tokens only in directives
|
|
|
|
yamlTagDirective, yamlYamlDirective, yamlUnknownDirective,
|
|
|
|
yamlMajorVersion, yamlMinorVersion, yamlTagURI,
|
|
|
|
yamlUnknownDirectiveParam,
|
|
|
|
# tokens in directives and content
|
|
|
|
yamlTagHandle, yamlComment,
|
|
|
|
# from here on tokens only in content
|
2015-11-29 21:27:05 +00:00
|
|
|
yamlLineStart,
|
|
|
|
# control characters
|
|
|
|
yamlColon, yamlDash, yamlQuestionmark, yamlComma, yamlOpeningBrace,
|
|
|
|
yamlOpeningBracket, yamlClosingBrace, yamlClosingBracket, yamlPipe,
|
|
|
|
yamlGreater,
|
2015-11-29 14:42:41 +00:00
|
|
|
# block scalar header
|
2015-11-27 22:42:11 +00:00
|
|
|
yamlLiteralScalar, yamlFoldedScalar,
|
2015-11-29 14:42:41 +00:00
|
|
|
yamlBlockIndentationIndicator, yamlBlockChompingIndicator,
|
|
|
|
# scalar content
|
|
|
|
yamlScalar, yamlBlockScalarLine,
|
|
|
|
# tags
|
|
|
|
yamlVerbatimTag, yamlTagSuffix,
|
|
|
|
# anchoring
|
|
|
|
yamlAnchor, yamlAlias,
|
|
|
|
# error reporting
|
|
|
|
yamlError
|
|
|
|
|
2015-11-27 22:42:11 +00:00
|
|
|
|
|
|
|
YamlLexerEvent* = tuple
|
|
|
|
kind: YamlLexerEventKind
|
2015-11-29 14:42:41 +00:00
|
|
|
position: int # X position relative to line start (0-based)
|
2015-11-27 22:42:11 +00:00
|
|
|
|
|
|
|
YamlLexerState = enum
|
2015-11-29 14:42:41 +00:00
|
|
|
# initial states (not started reading any token)
|
|
|
|
ylInitial, ylInitialSpaces, ylInitialUnknown, ylInitialContent,
|
2015-11-29 21:27:05 +00:00
|
|
|
ylDefineTagHandleInitial, ylDefineTagURIInitial, ylInitialInLine,
|
2015-11-29 14:42:41 +00:00
|
|
|
ylLineEnd, ylDirectiveLineEnd,
|
|
|
|
# directive reading states
|
|
|
|
ylDirective, ylDefineTagHandle, ylDefineTagURI, ylMajorVersion,
|
|
|
|
ylMinorVersion, ylUnknownDirectiveParam, ylDirectiveComment,
|
|
|
|
# scalar reading states
|
|
|
|
ylPlainScalar, ylSingleQuotedScalar, ylDoublyQuotedScalar,
|
2015-11-29 15:50:27 +00:00
|
|
|
ylEscape, ylBlockScalar, ylBlockScalarHeader,
|
|
|
|
ylSpaceAfterPlainScalar, ylSpaceAfterQuotedScalar,
|
2015-11-29 14:42:41 +00:00
|
|
|
# indentation
|
|
|
|
ylIndentation,
|
|
|
|
# comments
|
|
|
|
ylComment,
|
|
|
|
# tags
|
|
|
|
ylTagHandle, ylTagSuffix, ylVerbatimTag,
|
|
|
|
# document separation
|
|
|
|
ylDashes, ylDots
|
2015-11-27 22:42:11 +00:00
|
|
|
|
|
|
|
YamlLexer* = object of BaseLexer
|
|
|
|
indentations: seq[int]
|
|
|
|
encoding: Encoding
|
|
|
|
charlen: int
|
|
|
|
charoffset: int
|
2015-11-29 14:42:41 +00:00
|
|
|
content*: string # my.content of the last returned token.
|
2015-11-27 22:42:11 +00:00
|
|
|
|
|
|
|
const
|
2015-11-29 14:42:41 +00:00
|
|
|
UTF8NextLine = toUTF8(Rune(0x85))
|
|
|
|
UTF8NonBreakingSpace = toUTF8(Rune(0xA0))
|
|
|
|
UTF8LineSeparator = toUTF8(Rune(0x2028))
|
2015-11-27 22:42:11 +00:00
|
|
|
UTF8ParagraphSeparator = toUTF8(Rune(0x2029))
|
|
|
|
|
|
|
|
proc detect_encoding(my: var YamlLexer) =
|
|
|
|
var numBomChars = 0
|
|
|
|
my.encoding = Unsupported
|
|
|
|
if my.bufpos == 3:
|
|
|
|
# BaseLexer already skipped UTF-8 BOM
|
|
|
|
my.encoding = UTF8
|
|
|
|
else:
|
|
|
|
case my.buf[0]
|
|
|
|
of '\0':
|
|
|
|
if my.buf[1] == '\0':
|
|
|
|
if my.buf[2] == '\0':
|
|
|
|
my.encoding = UTF32LE
|
|
|
|
elif my.buf[2] == '\xFE' and my.buf[3] == '\xFF':
|
|
|
|
my.encoding = UTF32BE
|
|
|
|
numBomChars = 4
|
|
|
|
else:
|
|
|
|
# this is probably not a unicode character stream,
|
|
|
|
# but we just use the next match in the table
|
|
|
|
my.encoding = UTF16BE
|
|
|
|
else:
|
|
|
|
# this is how a BOM-less UTF16BE input should actually look like
|
|
|
|
my.encoding = UTF16BE
|
|
|
|
of '\xFF':
|
|
|
|
case my.buf[1]
|
|
|
|
of '\xFE':
|
|
|
|
if my.buf[2] == '\0' and my.buf[3] == '\0':
|
|
|
|
my.encoding = UTF32LE
|
|
|
|
numBomChars = 4
|
|
|
|
else:
|
|
|
|
my.encoding = UTF16LE
|
|
|
|
numBomChars = 2
|
|
|
|
of '\0':
|
|
|
|
my.encoding = UTF16LE
|
|
|
|
else:
|
|
|
|
my.encoding = UTF8
|
|
|
|
of '\xFE':
|
|
|
|
case my.buf[1]
|
|
|
|
of '\xFF':
|
|
|
|
my.encoding = UTF16BE
|
|
|
|
numBomChars = 2
|
|
|
|
of '\0':
|
|
|
|
my.encoding = UTF16LE
|
|
|
|
else:
|
|
|
|
my.encoding = UTF8
|
|
|
|
else:
|
|
|
|
if my.buf[1] == '\0':
|
|
|
|
my.encoding = UTF16LE
|
|
|
|
else:
|
|
|
|
my.encoding = UTF8
|
|
|
|
inc(my.bufPos, numBomChars)
|
|
|
|
my.charlen = case my.encoding
|
|
|
|
of UTF8, Unsupported: 1
|
|
|
|
of UTF16LE, UTF16BE: 2
|
|
|
|
of UTF32LE, UTF32BE: 4
|
|
|
|
my.charoffset = case my.encoding
|
|
|
|
of UTF8, Unsupported, UTF16LE, UTF32LE: 0
|
|
|
|
of UTF16BE: 1
|
|
|
|
of UTF32BE: 3
|
|
|
|
|
|
|
|
proc open*(my: var YamlLexer, input: Stream) =
|
|
|
|
lexbase.open(my, input)
|
|
|
|
my.indentations = newSeq[int]()
|
|
|
|
my.detect_encoding()
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = ""
|
2015-11-27 22:42:11 +00:00
|
|
|
|
|
|
|
template yieldToken(mKind: YamlLexerEventKind) {.dirty.} =
|
2015-11-29 14:42:41 +00:00
|
|
|
yield (kind: mKind, position: position)
|
|
|
|
my.content = ""
|
2015-11-27 22:42:11 +00:00
|
|
|
|
|
|
|
template yieldError(message: string) {.dirty.} =
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = message
|
|
|
|
yield (kind: yamlError, position: position)
|
|
|
|
my.content = ""
|
2015-11-27 22:42:11 +00:00
|
|
|
|
|
|
|
template handleCR() {.dirty.} =
|
|
|
|
my.bufpos = lexbase.handleLF(my, my.bufpos + my.charoffset) + my.charlen -
|
|
|
|
my.charoffset - 1
|
|
|
|
|
|
|
|
template handleLF() {.dirty.} =
|
|
|
|
my.bufpos = lexbase.handleLF(my, my.bufpos + my.charoffset) +
|
|
|
|
my.charlen - my.charoffset - 1
|
|
|
|
|
|
|
|
template `or`(r: Rune, i: int): Rune =
|
|
|
|
cast[Rune](cast[int](r) or i)
|
|
|
|
|
|
|
|
iterator tokens*(my: var YamlLexer): YamlLexerEvent =
|
|
|
|
var
|
2015-11-29 14:42:41 +00:00
|
|
|
# the following three values are used for parsing escaped unicode chars
|
|
|
|
|
2015-11-27 22:42:11 +00:00
|
|
|
unicodeChar: Rune = cast[Rune](0)
|
|
|
|
escapeLength = 0
|
|
|
|
expectedEscapeLength = 0
|
2015-11-29 14:42:41 +00:00
|
|
|
|
2015-11-29 15:50:27 +00:00
|
|
|
trailingSpace = ""
|
|
|
|
# used to temporarily store whitespace after a plain scalar
|
2015-11-27 22:42:11 +00:00
|
|
|
lastSpecialChar: char = '\0'
|
2015-11-29 14:42:41 +00:00
|
|
|
# stores chars that behave differently dependent on the following
|
|
|
|
# char. handling will be deferred to next loop iteration.
|
2015-11-27 22:42:11 +00:00
|
|
|
flowDepth = 0
|
2015-11-29 14:42:41 +00:00
|
|
|
# Lexer must know whether it parses block or flow style. Therefore,
|
|
|
|
# it counts the number of open flow arrays / maps here
|
|
|
|
state = ylInitial # lexer state
|
|
|
|
lastIndentationLength = 0
|
|
|
|
# after parsing the indentation of the line, this will hold the
|
|
|
|
# indentation length of the current line. Needed for checking where
|
|
|
|
# a block scalar ends.
|
|
|
|
blockScalarIndentation = -1
|
|
|
|
# when parsing a block scalar, this will be set to the indentation
|
|
|
|
# of the line that starts the flow scalar.
|
|
|
|
position = 0
|
|
|
|
# X position of the current char. This is needed in later processing
|
|
|
|
# stages for correctly matching items in nested arrays and such.
|
2015-11-27 22:42:11 +00:00
|
|
|
while true:
|
|
|
|
let c = my.buf[my.bufpos + my.charoffset]
|
|
|
|
case state
|
2015-11-29 14:42:41 +00:00
|
|
|
of ylInitial:
|
|
|
|
case c
|
|
|
|
of '%':
|
|
|
|
state = ylDirective
|
|
|
|
continue
|
|
|
|
of ' ', '\t':
|
|
|
|
state = ylInitialSpaces
|
|
|
|
continue
|
|
|
|
of '#':
|
|
|
|
state = ylDirectiveComment
|
|
|
|
else:
|
|
|
|
state = ylInitialContent
|
|
|
|
continue
|
|
|
|
of ylInitialSpaces:
|
|
|
|
case c
|
|
|
|
of ' ', '\t':
|
|
|
|
my.content.add(c)
|
|
|
|
of '#':
|
2015-11-29 15:50:27 +00:00
|
|
|
my.content = ""
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylDirectiveComment
|
2015-11-29 14:42:41 +00:00
|
|
|
of EndOfFile, '\r', '\x0A':
|
|
|
|
state = ylDirectiveLineEnd
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
state = ylIndentation
|
|
|
|
continue
|
|
|
|
of ylInitialContent:
|
|
|
|
case c
|
|
|
|
of '-':
|
|
|
|
state = ylDashes
|
|
|
|
continue
|
|
|
|
of '.':
|
2015-11-29 21:27:05 +00:00
|
|
|
yieldToken(yamlLineStart)
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylDots
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
state = ylIndentation
|
|
|
|
continue
|
|
|
|
of ylDashes:
|
|
|
|
case c
|
|
|
|
of '-':
|
|
|
|
my.content.add(c)
|
|
|
|
of ' ', '\t', '\r', '\x0A', EndOfFile:
|
|
|
|
case my.content.len
|
|
|
|
of 3:
|
|
|
|
yieldToken(yamlDirectivesEnd)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-29 14:42:41 +00:00
|
|
|
of 1:
|
2015-11-29 21:27:05 +00:00
|
|
|
my.content = ""
|
|
|
|
yieldToken(yamlLineStart)
|
2015-11-29 14:42:41 +00:00
|
|
|
lastSpecialChar = '-'
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-29 14:42:41 +00:00
|
|
|
else:
|
2015-11-29 21:27:05 +00:00
|
|
|
let tmp = my.content
|
|
|
|
my.content = ""
|
|
|
|
yieldToken(yamlLineStart)
|
|
|
|
my.content = tmp
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylPlainScalar
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
state = ylPlainScalar
|
|
|
|
continue
|
|
|
|
of ylDots:
|
|
|
|
case c
|
|
|
|
of '.':
|
|
|
|
my.content.add(c)
|
|
|
|
of ' ', '\t', '\r', '\x0A', EndOfFile:
|
|
|
|
case my.content.len
|
|
|
|
of 3:
|
|
|
|
yieldToken(yamlDocumentEnd)
|
|
|
|
state = ylDirectiveLineEnd
|
|
|
|
else:
|
|
|
|
state = ylPlainScalar
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
state = ylPlainScalar
|
|
|
|
continue
|
|
|
|
of ylDirectiveLineEnd:
|
|
|
|
case c
|
|
|
|
of '\r':
|
|
|
|
handleCR()
|
|
|
|
state = ylInitial
|
2015-11-29 15:50:27 +00:00
|
|
|
continue
|
2015-11-29 14:42:41 +00:00
|
|
|
of '\x0A':
|
|
|
|
handleLF()
|
|
|
|
state = ylInitial
|
2015-11-29 15:50:27 +00:00
|
|
|
continue
|
2015-11-29 14:42:41 +00:00
|
|
|
of EndOfFile:
|
|
|
|
yieldToken(yamlStreamEnd)
|
|
|
|
break
|
|
|
|
of ' ', '\t':
|
|
|
|
discard
|
|
|
|
of '#':
|
|
|
|
state = ylDirectiveComment
|
|
|
|
else:
|
|
|
|
yieldError("Unexpected content at end of directive: " & c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylLineEnd:
|
|
|
|
case c
|
|
|
|
of '\r':
|
|
|
|
handleCR()
|
|
|
|
of '\x0A':
|
|
|
|
handleLF()
|
|
|
|
of EndOfFile:
|
|
|
|
yieldToken(yamlStreamEnd)
|
|
|
|
break
|
|
|
|
else:
|
2015-11-29 15:50:27 +00:00
|
|
|
yieldError("Internal error: Unexpected char at line end: " & c)
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylInitialContent
|
|
|
|
position = 0
|
2015-11-29 15:50:27 +00:00
|
|
|
continue
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylSingleQuotedScalar:
|
|
|
|
if lastSpecialChar != '\0':
|
|
|
|
# ' is the only special char
|
|
|
|
case c
|
|
|
|
of '\'':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
lastSpecialChar = '\0'
|
|
|
|
of EndOfFile, '\r', '\x0A':
|
|
|
|
yieldToken(yamlScalar)
|
|
|
|
lastSpecialChar = '\0'
|
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
yieldToken(yamlScalar)
|
|
|
|
lastSpecialChar = '\0'
|
2015-11-29 15:50:27 +00:00
|
|
|
state = ylSpaceAfterQuotedScalar
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
else:
|
|
|
|
case c
|
|
|
|
of '\'':
|
|
|
|
lastSpecialChar = c
|
|
|
|
of EndOfFile:
|
|
|
|
yieldError("Unterminated single quoted string")
|
|
|
|
yieldToken(yamlStreamEnd)
|
|
|
|
break
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylDoublyQuotedScalar:
|
|
|
|
case c
|
|
|
|
of '"':
|
|
|
|
yieldToken(yamlScalar)
|
2015-11-29 15:50:27 +00:00
|
|
|
state = ylSpaceAfterQuotedScalar
|
2015-11-27 22:42:11 +00:00
|
|
|
of EndOfFile:
|
|
|
|
yieldError("Unterminated doubly quoted string")
|
|
|
|
yieldToken(yamlStreamEnd)
|
|
|
|
break
|
|
|
|
of '\\':
|
|
|
|
state = ylEscape
|
|
|
|
escapeLength = 0
|
|
|
|
of '\r':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add("\x0A")
|
2015-11-27 22:42:11 +00:00
|
|
|
handleCR()
|
|
|
|
of '\x0A':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
handleLF()
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylEscape:
|
|
|
|
if escapeLength == 0:
|
|
|
|
expectedEscapeLength = 0
|
|
|
|
case c
|
|
|
|
of EndOfFile:
|
|
|
|
yieldError("Unterminated doubly quoted string")
|
2015-11-29 14:42:41 +00:00
|
|
|
of '0': my.content.add('\0')
|
|
|
|
of 'a': my.content.add('\x07')
|
|
|
|
of 'b': my.content.add('\x08')
|
|
|
|
of '\t', 't': my.content.add('\t')
|
|
|
|
of 'n': my.content.add('\x0A')
|
|
|
|
of 'v': my.content.add('\v')
|
|
|
|
of 'f': my.content.add('\f')
|
|
|
|
of 'r': my.content.add('\r')
|
|
|
|
of 'e': my.content.add('\e')
|
|
|
|
of ' ': my.content.add(' ')
|
|
|
|
of '"': my.content.add('"')
|
|
|
|
of '/': my.content.add('/')
|
|
|
|
of '\\': my.content.add('\\')
|
|
|
|
of 'N': my.content.add(UTF8NextLine)
|
|
|
|
of '_': my.content.add(UTF8NonBreakingSpace)
|
|
|
|
of 'L': my.content.add(UTF8LineSeparator)
|
|
|
|
of 'P': my.content.add(UTF8ParagraphSeparator)
|
2015-11-27 22:42:11 +00:00
|
|
|
of 'x': unicodeChar = cast[Rune](0); expectedEscapeLength = 3
|
|
|
|
of 'u': unicodeChar = cast[Rune](0); expectedEscapeLength = 5
|
|
|
|
of 'U': unicodeChar = cast[Rune](0); expectedEscapeLength = 9
|
|
|
|
else:
|
|
|
|
yieldError("Unsupported escape sequence: \\" & c)
|
|
|
|
if expectedEscapeLength == 0: state = ylDoublyQuotedScalar
|
|
|
|
else:
|
2015-11-29 20:01:22 +00:00
|
|
|
let digitPosition = expectedEscapeLength - escapeLength - 1
|
2015-11-27 22:42:11 +00:00
|
|
|
case c
|
|
|
|
of EndOFFile:
|
|
|
|
yieldError("Unterminated escape sequence")
|
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
|
|
|
of '0' .. '9':
|
|
|
|
unicodeChar = unicodechar or
|
2015-11-29 20:01:22 +00:00
|
|
|
(cast[int](c) - 0x30) shl (digitPosition * 4)
|
2015-11-27 22:42:11 +00:00
|
|
|
of 'A' .. 'F':
|
|
|
|
unicodeChar = unicodechar or
|
2015-11-29 20:01:22 +00:00
|
|
|
(cast[int](c) - 0x37) shl (digitPosition * 4)
|
2015-11-27 22:42:11 +00:00
|
|
|
of 'a' .. 'f':
|
|
|
|
unicodeChar = unicodechar or
|
2015-11-29 20:01:22 +00:00
|
|
|
(cast[int](c) - 0x57) shl (digitPosition * 4)
|
2015-11-27 22:42:11 +00:00
|
|
|
else:
|
2015-11-29 20:01:22 +00:00
|
|
|
yieldError("unsupported char in unicode escape sequence: " &
|
|
|
|
c)
|
2015-11-27 22:42:11 +00:00
|
|
|
escapeLength = 0
|
|
|
|
state = ylDoublyQuotedScalar
|
|
|
|
continue
|
|
|
|
inc(escapeLength)
|
|
|
|
if escapeLength == expectedEscapeLength and escapeLength > 0:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(toUTF8(unicodeChar))
|
2015-11-27 22:42:11 +00:00
|
|
|
state = ylDoublyQuotedScalar
|
|
|
|
|
2015-11-29 15:50:27 +00:00
|
|
|
of ylSpaceAfterQuotedScalar:
|
|
|
|
case c
|
|
|
|
of ' ', '\t':
|
|
|
|
trailingSpace.add(c)
|
|
|
|
of '#':
|
|
|
|
if trailingSpace.len > 0:
|
|
|
|
yieldError("Missing space before comment start")
|
|
|
|
state = ylComment
|
|
|
|
trailingSpace = ""
|
|
|
|
else:
|
|
|
|
trailingSpace = ""
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-29 15:50:27 +00:00
|
|
|
continue
|
|
|
|
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylPlainScalar:
|
2015-11-29 15:50:27 +00:00
|
|
|
case c
|
|
|
|
of EndOfFile, '\r', '\x0A':
|
|
|
|
yieldToken(yamlScalar)
|
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
|
|
|
of ':':
|
|
|
|
lastSpecialChar = c
|
|
|
|
state = ylSpaceAfterPlainScalar
|
|
|
|
of ' ':
|
|
|
|
state = ylSpaceAfterPlainScalar
|
|
|
|
continue
|
|
|
|
of ',':
|
|
|
|
if flowDepth > 0: lastSpecialChar = c
|
|
|
|
else: my.content.add(c)
|
|
|
|
of '[', ']', '{', '}':
|
|
|
|
yieldToken(yamlScalar)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-29 15:50:27 +00:00
|
|
|
continue
|
|
|
|
else:
|
|
|
|
my.content.add(c)
|
|
|
|
|
|
|
|
of ylSpaceAfterPlainScalar:
|
2015-11-27 22:42:11 +00:00
|
|
|
if lastSpecialChar != '\0':
|
|
|
|
case c
|
|
|
|
of ' ', '\t', EndOfFile, '\r', '\x0A':
|
|
|
|
yieldToken(yamlScalar)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
else:
|
2015-11-29 15:50:27 +00:00
|
|
|
my.content.add(trailingSpace)
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(lastSpecialChar)
|
2015-11-27 22:42:11 +00:00
|
|
|
lastSpecialChar = '\0'
|
2015-11-29 15:50:27 +00:00
|
|
|
trailingSpace = ""
|
|
|
|
state = ylPlainScalar
|
|
|
|
continue
|
2015-11-27 22:42:11 +00:00
|
|
|
|
|
|
|
case c
|
|
|
|
of EndOfFile, '\r', '\x0A':
|
2015-11-29 15:50:27 +00:00
|
|
|
trailingSpace = ""
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlScalar)
|
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
2015-11-29 15:50:27 +00:00
|
|
|
of ' ', '\t':
|
|
|
|
trailingSpace.add(c)
|
|
|
|
of ',':
|
|
|
|
if flowDepth > 0:
|
|
|
|
lastSpecialChar = c
|
|
|
|
else:
|
|
|
|
my.content.add(trailingSpace)
|
|
|
|
my.content.add(c)
|
|
|
|
trailingSpace = ""
|
|
|
|
state = ylPlainScalar
|
2015-11-27 22:42:11 +00:00
|
|
|
of ':', '#':
|
|
|
|
lastSpecialChar = c
|
|
|
|
of '[', ']', '{', '}':
|
|
|
|
yieldToken(yamlScalar)
|
2015-11-29 15:50:27 +00:00
|
|
|
trailingSpace = ""
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
else:
|
2015-11-29 15:50:27 +00:00
|
|
|
my.content.add(trailingSpace)
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-29 15:50:27 +00:00
|
|
|
trailingSpace = ""
|
|
|
|
state = ylPlainScalar
|
2015-11-27 22:42:11 +00:00
|
|
|
|
2015-11-29 21:27:05 +00:00
|
|
|
of ylInitialInLine:
|
2015-11-27 22:42:11 +00:00
|
|
|
if lastSpecialChar != '\0':
|
|
|
|
case c
|
|
|
|
of ' ', '\t', '\r', '\x0A', EndOfFile:
|
|
|
|
case lastSpecialChar
|
|
|
|
of '#':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = "#"
|
2015-11-27 22:42:11 +00:00
|
|
|
state = ylComment
|
2015-11-29 21:27:05 +00:00
|
|
|
of ':':
|
|
|
|
yieldToken(yamlColon)
|
|
|
|
of '?':
|
|
|
|
yieldToken(yamlQuestionmark)
|
|
|
|
of '-':
|
|
|
|
yieldToken(yamlDash)
|
2015-11-27 22:42:11 +00:00
|
|
|
else:
|
2015-11-29 21:27:05 +00:00
|
|
|
yieldError("Unexpected special char: \"" &
|
|
|
|
lastSpecialChar & "\"")
|
|
|
|
lastSpecialChar = '\0'
|
2015-11-27 22:42:11 +00:00
|
|
|
elif lastSpecialChar == '!':
|
|
|
|
case c
|
|
|
|
of '<':
|
|
|
|
state = ylVerbatimTag
|
|
|
|
lastSpecialChar = '\0'
|
|
|
|
my.bufpos += my.charlen
|
|
|
|
else:
|
|
|
|
state = ylTagHandle
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = "!"
|
2015-11-27 22:42:11 +00:00
|
|
|
lastSpecialChar = '\0'
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(lastSpecialChar)
|
2015-11-27 22:42:11 +00:00
|
|
|
lastSpecialChar = '\0'
|
|
|
|
state = ylPlainScalar
|
|
|
|
continue
|
|
|
|
case c
|
2015-11-29 15:50:27 +00:00
|
|
|
of '\r', '\x0A', EndOfFile:
|
2015-11-27 22:42:11 +00:00
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
|
|
|
of ',':
|
2015-11-29 21:27:05 +00:00
|
|
|
if flowDepth > 0:
|
|
|
|
yieldToken(yamlComma)
|
2015-11-27 22:42:11 +00:00
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = "" & c
|
2015-11-27 22:42:11 +00:00
|
|
|
state = ylPlainScalar
|
2015-11-29 21:27:05 +00:00
|
|
|
of '[':
|
2015-11-27 22:42:11 +00:00
|
|
|
inc(flowDepth)
|
2015-11-29 21:27:05 +00:00
|
|
|
yieldToken(yamlOpeningBracket)
|
|
|
|
of '{':
|
|
|
|
inc(flowDepth)
|
|
|
|
yieldToken(yamlOpeningBrace)
|
|
|
|
of ']':
|
|
|
|
yieldToken(yamlClosingBracket)
|
|
|
|
if flowDepth > 0:
|
|
|
|
inc(flowDepth, -1)
|
|
|
|
of '}':
|
|
|
|
yieldToken(yamlClosingBrace)
|
|
|
|
if flowDepth > 0:
|
2015-11-27 22:42:11 +00:00
|
|
|
inc(flowDepth, -1)
|
|
|
|
of '#':
|
|
|
|
lastSpecialChar = '#'
|
|
|
|
of '"':
|
|
|
|
state = ylDoublyQuotedScalar
|
|
|
|
of '\'':
|
|
|
|
state = ylSingleQuotedScalar
|
|
|
|
of '!':
|
2015-11-29 21:27:05 +00:00
|
|
|
my.content.add(c)
|
|
|
|
state = ylTagHandle
|
2015-11-27 22:42:11 +00:00
|
|
|
of '&':
|
|
|
|
yieldError("TODO: anchors")
|
|
|
|
of '*':
|
|
|
|
yieldError("TODO: links")
|
|
|
|
of ' ':
|
|
|
|
discard
|
|
|
|
of '-':
|
2015-11-29 21:27:05 +00:00
|
|
|
if flowDepth == 0:
|
2015-11-27 22:42:11 +00:00
|
|
|
lastSpecialChar = '-'
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = "" & c
|
2015-11-27 22:42:11 +00:00
|
|
|
state = ylPlainScalar
|
|
|
|
of '?', ':':
|
|
|
|
lastSpecialChar = c
|
2015-11-29 21:27:05 +00:00
|
|
|
of '|':
|
|
|
|
yieldToken(yamlPipe)
|
|
|
|
state = ylBlockScalarHeader
|
|
|
|
of '>':
|
|
|
|
yieldToken(yamlGreater)
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylBlockScalarHeader
|
2015-11-27 22:42:11 +00:00
|
|
|
of '\t':
|
|
|
|
discard
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = "" & c
|
2015-11-27 22:42:11 +00:00
|
|
|
state = ylPlainScalar
|
2015-11-29 14:42:41 +00:00
|
|
|
of ylComment, ylDirectiveComment:
|
2015-11-27 22:42:11 +00:00
|
|
|
case c
|
|
|
|
of EndOfFile, '\r', '\x0A':
|
|
|
|
yieldToken(yamlComment)
|
2015-11-29 14:42:41 +00:00
|
|
|
case state
|
|
|
|
of ylComment:
|
|
|
|
state = ylLineEnd
|
|
|
|
of ylDirectiveComment:
|
|
|
|
state = ylDirectiveLineEnd
|
|
|
|
else:
|
|
|
|
yieldError("Should never happen")
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
|
|
|
of ylIndentation:
|
2015-11-27 22:42:11 +00:00
|
|
|
case c
|
|
|
|
of EndOfFile, '\r', '\x0A':
|
2015-11-29 14:42:41 +00:00
|
|
|
lastIndentationLength = my.content.len
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlLineStart)
|
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
|
|
|
of ' ':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(' ')
|
2015-11-27 22:42:11 +00:00
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
lastIndentationLength = my.content.len
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlLineStart)
|
2015-11-29 14:42:41 +00:00
|
|
|
if blockScalarIndentation != -1:
|
|
|
|
if lastIndentationLength <= blockScalarIndentation:
|
|
|
|
blockScalarIndentation = -1
|
|
|
|
else:
|
|
|
|
state = ylBlockScalar
|
|
|
|
continue
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
of ylTagHandle:
|
|
|
|
case c
|
|
|
|
of '!':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlTagHandle)
|
|
|
|
state = ylTagSuffix
|
|
|
|
of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '-':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of ' ', '\t', EndOfFile, '\r', '\x0A':
|
2015-11-29 14:42:41 +00:00
|
|
|
var suffix = my.content[1..^1]
|
|
|
|
my.content = "!"
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlTagHandle)
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = suffix
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlTagSuffix)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-29 15:50:27 +00:00
|
|
|
continue
|
2015-11-27 22:42:11 +00:00
|
|
|
else:
|
|
|
|
yieldError("Invalid character in tag handle: " & c)
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = ""
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylTagSuffix:
|
|
|
|
case c
|
|
|
|
of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', '@',
|
|
|
|
'&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of ' ', '\t', EndOfFile, '\r', '\x0A':
|
|
|
|
yieldToken(yamlTagSuffix)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-29 15:50:27 +00:00
|
|
|
continue
|
2015-11-27 22:42:11 +00:00
|
|
|
else:
|
|
|
|
yieldError("Invalid character in tag suffix: " & c)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylVerbatimTag:
|
|
|
|
case c
|
|
|
|
of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', '@',
|
|
|
|
'&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of '>':
|
|
|
|
yieldToken(yamlVerbatimTag)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
of EndOfFile, '\r', '\x0A':
|
|
|
|
yieldError("Unfinished verbatim tag")
|
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
yieldError("Invalid character in tag URI: " & c)
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = ""
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylDirective:
|
|
|
|
case c
|
|
|
|
of ' ', '\t', '\r', '\x0A', EndOfFile:
|
2015-11-29 14:42:41 +00:00
|
|
|
if my.content == "%YAML":
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlYamlDirective)
|
|
|
|
state = ylMajorVersion
|
2015-11-29 14:42:41 +00:00
|
|
|
elif my.content == "%TAG":
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlTagDirective)
|
|
|
|
state = ylDefineTagHandleInitial
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
yieldToken(yamlUnknownDirective)
|
|
|
|
state = ylInitialUnknown
|
2015-11-27 22:42:11 +00:00
|
|
|
if c == EndOfFile:
|
|
|
|
continue
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
|
|
|
of ylInitialUnknown:
|
|
|
|
case c
|
|
|
|
of ' ', '\t':
|
|
|
|
discard
|
|
|
|
of '\r', '\x0A', EndOfFile:
|
|
|
|
state = ylDirectiveLineEnd
|
|
|
|
continue
|
|
|
|
of '#':
|
|
|
|
state = ylDirectiveComment
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
state = ylUnknownDirectiveParam
|
2015-11-29 15:50:27 +00:00
|
|
|
continue
|
2015-11-29 14:42:41 +00:00
|
|
|
of ylUnknownDirectiveParam:
|
|
|
|
case c
|
|
|
|
of '\r', '\x0A', EndOfFile, ' ', '\t':
|
|
|
|
yieldToken(yamlUnknownDirectiveParam)
|
|
|
|
state = ylInitialUnknown
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
my.content.add(c)
|
|
|
|
of ylMajorVersion:
|
2015-11-27 22:42:11 +00:00
|
|
|
case c
|
|
|
|
of '0' .. '9':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of '.':
|
2015-11-29 14:42:41 +00:00
|
|
|
yieldToken(yamlMajorVersion)
|
|
|
|
state = ylMinorVersion
|
|
|
|
of EndOfFile, '\r', '\x0A', ' ', '\t':
|
|
|
|
yieldError("Missing YAML minor version.")
|
|
|
|
state = ylDirectiveLineEnd
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
else:
|
|
|
|
yieldError("Invalid character in YAML version: " & c)
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylInitialUnknown
|
|
|
|
of ylMinorVersion:
|
|
|
|
case c
|
|
|
|
of '0' .. '9':
|
|
|
|
my.content.add(c)
|
|
|
|
of EndOfFile, '\r', '\x0A', ' ', '\t':
|
|
|
|
yieldToken(yamlMinorVersion)
|
|
|
|
state = ylDirectiveLineEnd
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
yieldError("Invalid character in YAML version: " & c)
|
|
|
|
state = ylInitialUnknown
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylDefineTagHandleInitial:
|
|
|
|
case c
|
|
|
|
of ' ', '\t':
|
|
|
|
discard
|
|
|
|
of EndOfFile, '\r', '\x0A':
|
|
|
|
yieldError("Unfinished %TAG directive")
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylDirectiveLineEnd
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
of '!':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
state = ylDefineTagHandle
|
|
|
|
else:
|
|
|
|
yieldError("Unexpected character in %TAG directive: " & c)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylDefineTagHandle:
|
|
|
|
case c
|
|
|
|
of '!':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlTagHandle)
|
|
|
|
state = ylDefineTagURIInitial
|
|
|
|
of 'a' .. 'z', 'A' .. 'Z', '-':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of EndOfFile, '\r', '\x0A':
|
|
|
|
yieldError("Unfinished %TAG directive")
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylDirectiveLineEnd
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
else:
|
|
|
|
yieldError("Unexpected char in %TAG directive: " & c)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
of ylDefineTagURIInitial:
|
|
|
|
case c
|
|
|
|
of '\t', ' ':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
of '\x0A', '\r', EndOfFile:
|
|
|
|
yieldError("Unfinished %TAG directive")
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylDirectiveLineEnd
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
else:
|
2015-11-29 14:42:41 +00:00
|
|
|
if my.content.len == 0:
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldError("Missing whitespace in %TAG directive")
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content = ""
|
2015-11-27 22:42:11 +00:00
|
|
|
state = ylDefineTagURI
|
|
|
|
continue
|
|
|
|
of ylDefineTagURI:
|
|
|
|
case c
|
|
|
|
of 'a' .. 'z', 'A' .. 'Z', '0' .. '9', '#', ';', '/', '?', ':', '@',
|
|
|
|
'&', '=', '+', '$', ',', '_', '.', '~', '*', '\'', '(', ')':
|
2015-11-29 14:42:41 +00:00
|
|
|
my.content.add(c)
|
|
|
|
of '\x0A', '\r', EndOfFile, ' ', '\t':
|
2015-11-27 22:42:11 +00:00
|
|
|
yieldToken(yamlTagURI)
|
2015-11-29 14:42:41 +00:00
|
|
|
state = ylDirectiveLineEnd
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
|
|
|
else:
|
|
|
|
yieldError("Invalid URI character: " & c)
|
2015-11-29 21:27:05 +00:00
|
|
|
state = ylInitialInLine
|
2015-11-27 22:42:11 +00:00
|
|
|
continue
|
2015-11-29 14:42:41 +00:00
|
|
|
of ylBlockScalarHeader:
|
|
|
|
case c
|
|
|
|
of '0' .. '9':
|
|
|
|
my.content = "" & c
|
|
|
|
yieldToken(yamlBlockIndentationIndicator)
|
|
|
|
of '+', '-':
|
|
|
|
my.content = "" & c
|
|
|
|
yieldToken(yamlBlockChompingIndicator)
|
|
|
|
of '\r', '\x0A', EndOfFile:
|
|
|
|
blockScalarIndentation = lastIndentationLength
|
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
yieldError("Unexpected character in block scalar header: " & c)
|
|
|
|
of ylBlockScalar:
|
|
|
|
case c
|
|
|
|
of EndOfFile, '\r', '\x0A':
|
|
|
|
yieldToken(yamlBlockScalarLine)
|
|
|
|
state = ylLineEnd
|
|
|
|
continue
|
|
|
|
else:
|
|
|
|
my.content.add(c)
|
2015-11-27 22:42:11 +00:00
|
|
|
|
2015-11-29 14:42:41 +00:00
|
|
|
my.bufpos += my.charlen
|
|
|
|
inc(position)
|