status-im
/
NimYAML
mirror of https://github.com/status-im/NimYAML.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactoring, less heap objects 

* YamlParserEvent -> YamlStreamEvent
 * YamlStreamEvent is no longer a `ref` object. Test suite now
   runs in a small fraction of a second (compared to several
   seconds before)
 * Introduced YamlStream
This commit is contained in:
Felix Krause 2015-12-26 13:39:43 +01:00
parent 2c4e681f0b
commit ca0f5d1741
4 changed files with 51 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
src/private/sequential.nim
View File

@ -49,7 +49,7 @@ proc anchor*(parser: YamlSequentialParser, id: AnchorId): string =
return pair[0] return pair[0]
return nil return nil
proc `==`*(left: YamlParserEvent, right: YamlParserEvent): bool = proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool =
if left.kind != right.kind: if left.kind != right.kind:
return false return false
case left.kind case left.kind
@ -70,11 +70,11 @@ proc `==`*(left: YamlParserEvent, right: YamlParserEvent): bool =
left.line == right.line and left.column == right.column left.line == right.line and left.column == right.column
template yieldWarning(d: string) {.dirty.} = template yieldWarning(d: string) {.dirty.} =
yield YamlParserEvent(kind: yamlWarning, description: d, yield YamlStreamEvent(kind: yamlWarning, description: d,
line: lex.line, column: lex.column) line: lex.line, column: lex.column)
template yieldError(d: string) {.dirty.} = template yieldError(d: string) {.dirty.} =
yield YamlParserEvent(kind: yamlError, description: d, yield YamlStreamEvent(kind: yamlError, description: d,
line: lex.line, column: lex.column) line: lex.line, column: lex.column)
break parserLoop break parserLoop
@ -117,20 +117,20 @@ template yieldScalar(content: string, typeHint: YamlTypeHint,
when defined(yamlDebug): when defined(yamlDebug):
echo "Parser token [mode=", level.mode, ", state=", state, "]: ", echo "Parser token [mode=", level.mode, ", state=", state, "]: ",
"scalar[\"", content, "\", type=", typeHint, "]" "scalar[\"", content, "\", type=", typeHint, "]"
yield YamlParserEvent(kind: yamlScalar, yield YamlStreamEvent(kind: yamlScalar,
scalarAnchor: resolveAnchor(parser, anchor), scalarAnchor: resolveAnchor(parser, anchor),
scalarTag: resolveTag(parser, tag, quoted), scalarTag: resolveTag(parser, tag, quoted),
scalarContent: content, scalarContent: content,
scalarType: typeHint) scalarType: typeHint)
template yieldStart(k: YamlParserEventKind) {.dirty.} = template yieldStart(k: YamlStreamEventKind) {.dirty.} =
when defined(yamlDebug): when defined(yamlDebug):
echo "Parser token [mode=", level.mode, ", state=", state, "]: ", k echo "Parser token [mode=", level.mode, ", state=", state, "]: ", k
yield YamlParserEvent(kind: k, objAnchor: resolveAnchor(parser, anchor), yield YamlStreamEvent(kind: k, objAnchor: resolveAnchor(parser, anchor),
objTag: resolveTag(parser, tag)) objTag: resolveTag(parser, tag))
template yieldDocumentEnd() {.dirty.} = template yieldDocumentEnd() {.dirty.} =
yield YamlParserEvent(kind: yamlEndDocument) yield YamlStreamEvent(kind: yamlEndDocument)
tagShorthands = initTable[string, string]() tagShorthands = initTable[string, string]()
tagShorthands["!"] = "!" tagShorthands["!"] = "!"
tagShorthands["!!"] = "tag:yaml.org,2002:" tagShorthands["!!"] = "tag:yaml.org,2002:"
@ -140,16 +140,16 @@ template closeLevel(lvl: DocumentLevel) {.dirty.} =
case lvl.mode case lvl.mode
of mExplicitBlockMapKey, mFlowMapKey: of mExplicitBlockMapKey, mFlowMapKey:
yieldScalar("", yTypeUnknown) yieldScalar("", yTypeUnknown)
yield YamlParserEvent(kind: yamlEndMap) yield YamlStreamEvent(kind: yamlEndMap)
of mImplicitBlockMapKey, mBlockMapValue, mFlowMapValue: of mImplicitBlockMapKey, mBlockMapValue, mFlowMapValue:
yield YamlParserEvent(kind: yamlEndMap) yield YamlStreamEvent(kind: yamlEndMap)
of mBlockSequenceItem, mFlowSequenceItem: of mBlockSequenceItem, mFlowSequenceItem:
yield YamlParserEvent(kind: yamlEndSequence) yield YamlStreamEvent(kind: yamlEndSequence)
of mScalar: of mScalar:
when defined(yamlDebug): when defined(yamlDebug):
echo "Parser token [mode=", level.mode, ", state=", state, "]: ", echo "Parser token [mode=", level.mode, ", state=", state, "]: ",
"scalar[\"", scalarCache, "\", type=", scalarCacheType, "]" "scalar[\"", scalarCache, "\", type=", scalarCacheType, "]"
yield YamlParserEvent(kind: yamlScalar, yield YamlStreamEvent(kind: yamlScalar,
scalarAnchor: resolveAnchor(parser, anchor), scalarAnchor: resolveAnchor(parser, anchor),
scalarTag: resolveTag(parser, tag), scalarTag: resolveTag(parser, tag),
scalarContent: scalarCache, scalarContent: scalarCache,
@ -188,7 +188,7 @@ template closeAllLevels() {.dirty.} =
template handleBlockIndicator(expected, possible: openarray[DocumentLevelMode], template handleBlockIndicator(expected, possible: openarray[DocumentLevelMode],
next: DocumentLevelMode, next: DocumentLevelMode,
entering: YamlParserEventKind, entering: YamlStreamEventKind,
emptyScalarOnOpening: bool = false) {.dirty.} = emptyScalarOnOpening: bool = false) {.dirty.} =
leaveMoreIndentedLevels() leaveMoreIndentedLevels()
if level.indicatorColumn == lex.column or if level.indicatorColumn == lex.column or
@ -259,9 +259,8 @@ template handleTagHandle() {.dirty.} =
else: else:
yieldError("Unknown tag shorthand: " & handle) yieldError("Unknown tag shorthand: " & handle)
proc parse*(parser: YamlSequentialParser, proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream =
s: Stream): iterator(): YamlParserEvent = result = iterator(): YamlStreamEvent =
result = iterator(): YamlParserEvent =
var var
# parsing state # parsing state
lex: YamlLexer lex: YamlLexer
@ -341,15 +340,15 @@ proc parse*(parser: YamlSequentialParser,
of tComment: of tComment:
discard discard
of tDirectivesEnd: of tDirectivesEnd:
yield YamlParserEvent(kind: yamlStartDocument) yield YamlStreamEvent(kind: yamlStartDocument)
level = DocumentLevel(mode: mUnknown, indicatorColumn: -1, level = DocumentLevel(mode: mUnknown, indicatorColumn: -1,
indentationColumn: -1) indentationColumn: -1)
state = ypAfterDirectivesEnd state = ypAfterDirectivesEnd
of tDocumentEnd, tStreamEnd: of tDocumentEnd, tStreamEnd:
yield YamlParserEvent(kind: yamlStartDocument) yield YamlStreamEvent(kind: yamlStartDocument)
yieldDocumentEnd() yieldDocumentEnd()
else: else:
yield YamlParserEvent(kind: yamlStartDocument) yield YamlStreamEvent(kind: yamlStartDocument)
state = ypBlockLineStart state = ypBlockLineStart
continue continue
of ypSkipDirective: of ypSkipDirective:
@ -456,7 +455,7 @@ proc parse*(parser: YamlSequentialParser,
yieldError("Unexpected alias") yieldError("Unexpected alias")
of tStreamEnd: of tStreamEnd:
closeAllLevels() closeAllLevels()
yield YamlParserEvent(kind: yamlEndDocument) yield YamlStreamEvent(kind: yamlEndDocument)
break break
of tDocumentEnd: of tDocumentEnd:
closeAllLevels() closeAllLevels()
@ -509,7 +508,7 @@ proc parse*(parser: YamlSequentialParser,
assert level.mode in [mUnknown, mImplicitBlockMapKey, mScalar] assert level.mode in [mUnknown, mImplicitBlockMapKey, mScalar]
if level.mode in [mUnknown, mScalar]: if level.mode in [mUnknown, mScalar]:
# tags and anchors are for key scalar, not for map. # tags and anchors are for key scalar, not for map.
yield YamlParserEvent(kind: yamlStartMap, yield YamlStreamEvent(kind: yamlStartMap,
objAnchor: anchorNone, objAnchor: anchorNone,
objTag: tagQuestionMark) objTag: tagQuestionMark)
level.mode = mBlockMapValue level.mode = mBlockMapValue
@ -538,7 +537,7 @@ proc parse*(parser: YamlSequentialParser,
if ancestry.len > 0: if ancestry.len > 0:
level = ancestry.pop() level = ancestry.pop()
closeAllLevels() closeAllLevels()
yield YamlParserEvent(kind: yamlEndDocument) yield YamlStreamEvent(kind: yamlEndDocument)
break break
else: else:
yieldUnexpectedToken() yieldUnexpectedToken()
@ -547,14 +546,14 @@ proc parse*(parser: YamlSequentialParser,
of tColon: of tColon:
assert level.mode in [mUnknown, mImplicitBlockMapKey] assert level.mode in [mUnknown, mImplicitBlockMapKey]
if level.mode == mUnknown: if level.mode == mUnknown:
yield YamlParserEvent(kind: yamlStartMap, yield YamlStreamEvent(kind: yamlStartMap,
objAnchor: anchorNone, objAnchor: anchorNone,
objTag: tagQuestionMark) objTag: tagQuestionMark)
level.mode = mBlockMapValue level.mode = mBlockMapValue
ancestry.add(level) ancestry.add(level)
level = DocumentLevel(mode: mUnknown, indicatorColumn: -1, level = DocumentLevel(mode: mUnknown, indicatorColumn: -1,
indentationColumn: -1) indentationColumn: -1)
yield YamlParserEvent(kind: yamlAlias, aliasTarget: aliasCache) yield YamlStreamEvent(kind: yamlAlias, aliasTarget: aliasCache)
state = ypBlockAfterColon state = ypBlockAfterColon
of tLineStart: of tLineStart:
if level.mode == mImplicitBlockMapKey: if level.mode == mImplicitBlockMapKey:
@ -562,10 +561,10 @@ proc parse*(parser: YamlSequentialParser,
if level.mode == mUnknown: if level.mode == mUnknown:
assert ancestry.len > 0 assert ancestry.len > 0
level = ancestry.pop() level = ancestry.pop()
yield YamlParserEvent(kind: yamlAlias, aliasTarget: aliasCache) yield YamlStreamEvent(kind: yamlAlias, aliasTarget: aliasCache)
state = ypBlockLineStart state = ypBlockLineStart
of tStreamEnd: of tStreamEnd:
yield YamlParserEvent(kind: yamlAlias, aliasTarget: aliasCache) yield YamlStreamEvent(kind: yamlAlias, aliasTarget: aliasCache)
if level.mode == mUnknown: if level.mode == mUnknown:
assert ancestry.len > 0 assert ancestry.len > 0
level = ancestry.pop() level = ancestry.pop()
@ -651,7 +650,7 @@ proc parse*(parser: YamlSequentialParser,
state = ypBlockLineStart state = ypBlockLineStart
of tStreamEnd: of tStreamEnd:
closeAllLevels() closeAllLevels()
yield YamlParserEvent(kind: yamlEndDocument) yield YamlStreamEvent(kind: yamlEndDocument)
break break
of tOpeningBracket, tOpeningBrace: of tOpeningBracket, tOpeningBrace:
state = ypFlow state = ypFlow
@ -679,7 +678,7 @@ proc parse*(parser: YamlSequentialParser,
if noAnchor: if noAnchor:
# cannot use yield within try/except, so do it here # cannot use yield within try/except, so do it here
yieldError("[alias] Unknown anchor: " & lex.content) yieldError("[alias] Unknown anchor: " & lex.content)
yield YamlParserEvent(kind: yamlAlias, aliasTarget: aliasCache) yield YamlStreamEvent(kind: yamlAlias, aliasTarget: aliasCache)
level = ancestry.pop() level = ancestry.pop()
state = ypBlockLineEnd state = ypBlockLineEnd
else: else:
@ -691,7 +690,7 @@ proc parse*(parser: YamlSequentialParser,
ypBlockLineStart ypBlockLineStart
of tStreamEnd: of tStreamEnd:
closeAllLevels() closeAllLevels()
yield YamlParserEvent(kind: yamlEndDocument) yield YamlStreamEvent(kind: yamlEndDocument)
break break
else: else:
yieldUnexpectedToken("line end") yieldUnexpectedToken("line end")
@ -830,7 +829,7 @@ proc parse*(parser: YamlSequentialParser,
level = ancestry.pop() level = ancestry.pop()
if level.mode != mFlowMapValue: if level.mode != mFlowMapValue:
yieldUnexpectedToken() yieldUnexpectedToken()
yield YamlParserEvent(kind: yamlEndMap) yield YamlStreamEvent(kind: yamlEndMap)
if ancestry.len > 0: if ancestry.len > 0:
level = ancestry.pop() level = ancestry.pop()
case level.mode case level.mode
@ -847,7 +846,7 @@ proc parse*(parser: YamlSequentialParser,
if level.mode != mFlowSequenceItem: if level.mode != mFlowSequenceItem:
yieldUnexpectedToken() yieldUnexpectedToken()
else: else:
yield YamlParserEvent(kind: yamlEndSequence) yield YamlStreamEvent(kind: yamlEndSequence)
if ancestry.len > 0: if ancestry.len > 0:
level = ancestry.pop() level = ancestry.pop()
case level.mode case level.mode
@ -864,7 +863,7 @@ proc parse*(parser: YamlSequentialParser,
anchor = lex.content anchor = lex.content
state = ypFlowAfterAnchor state = ypFlowAfterAnchor
of tAlias: of tAlias:
yield YamlParserEvent(kind: yamlAlias, yield YamlStreamEvent(kind: yamlAlias,
aliasTarget: resolveAlias(parser, lex.content)) aliasTarget: resolveAlias(parser, lex.content))
state = ypFlowAfterObject state = ypFlowAfterObject
level = ancestry.pop() level = ancestry.pop()
@ -931,7 +930,7 @@ proc parse*(parser: YamlSequentialParser,
if level.mode != mFlowMapValue: if level.mode != mFlowMapValue:
yieldUnexpectedToken() yieldUnexpectedToken()
else: else:
yield YamlParserEvent(kind: yamlEndMap) yield YamlStreamEvent(kind: yamlEndMap)
if ancestry.len > 0: if ancestry.len > 0:
level = ancestry.pop() level = ancestry.pop()
case level.mode case level.mode
@ -945,7 +944,7 @@ proc parse*(parser: YamlSequentialParser,
if level.mode != mFlowSequenceItem: if level.mode != mFlowSequenceItem:
yieldUnexpectedToken() yieldUnexpectedToken()
else: else:
yield YamlParserEvent(kind: yamlEndSequence) yield YamlStreamEvent(kind: yamlEndSequence)
if ancestry.len > 0: if ancestry.len > 0:
level = ancestry.pop() level = ancestry.pop()
case level.mode case level.mode

13
src/yaml.nim
View File

@ -5,7 +5,7 @@ type
yTypeInteger, yTypeFloat, yTypeBoolean, yTypeNull, yTypeString, yTypeInteger, yTypeFloat, yTypeBoolean, yTypeNull, yTypeString,
yTypeUnknown yTypeUnknown
YamlParserEventKind* = enum YamlStreamEventKind* = enum
yamlStartDocument, yamlEndDocument, yamlStartMap, yamlEndMap, yamlStartDocument, yamlEndDocument, yamlStartMap, yamlEndMap,
yamlStartSequence, yamlEndSequence, yamlScalar, yamlAlias, yamlStartSequence, yamlEndSequence, yamlScalar, yamlAlias,
yamlError, yamlWarning yamlError, yamlWarning
@ -13,8 +13,8 @@ type
TagId* = distinct int TagId* = distinct int
AnchorId* = distinct int AnchorId* = distinct int
YamlParserEvent* = ref object YamlStreamEvent* = object
case kind*: YamlParserEventKind case kind*: YamlStreamEventKind
of yamlStartMap, yamlStartSequence: of yamlStartMap, yamlStartSequence:
objAnchor* : AnchorId objAnchor* : AnchorId
objTag* : TagId objTag* : TagId
@ -32,6 +32,8 @@ type
line* : int line* : int
column* : int column* : int
YamlStream* = iterator(): YamlStreamEvent
YamlSequentialParser* = ref object YamlSequentialParser* = ref object
tags: OrderedTable[string, TagId] tags: OrderedTable[string, TagId]
anchors: OrderedTable[string, AnchorId] anchors: OrderedTable[string, AnchorId]
@ -43,7 +45,7 @@ const
# interface # interface
proc `==`*(left: YamlParserEvent, right: YamlParserEvent): bool proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool
proc `==`*(left, right: TagId): bool {.borrow.} proc `==`*(left, right: TagId): bool {.borrow.}
proc `$`*(id: TagId): string {.borrow.} proc `$`*(id: TagId): string {.borrow.}
@ -61,8 +63,7 @@ proc registerUri*(parser: var YamlSequentialParser, uri: string): TagId
proc anchor*(parser: YamlSequentialParser, id: AnchorId): string proc anchor*(parser: YamlSequentialParser, id: AnchorId): string
proc parse*(parser: YamlSequentialParser, s: Stream): proc parse*(parser: YamlSequentialParser, s: Stream): YamlStream
iterator(): YamlParserEvent
proc parseToJson*(s: Stream): seq[JsonNode] proc parseToJson*(s: Stream): seq[JsonNode]
proc parseToJson*(s: string): seq[JsonNode] proc parseToJson*(s: string): seq[JsonNode]

30
test/parsing.nim
View File

@ -3,18 +3,15 @@ import streams
import unittest import unittest
proc startDoc(): YamlParserEvent = proc startDoc(): YamlStreamEvent =
new(result)
result.kind = yamlStartDocument result.kind = yamlStartDocument
proc endDoc(): YamlParserEvent = proc endDoc(): YamlStreamEvent =
new(result)
result.kind = yamlEndDocument result.kind = yamlEndDocument
proc scalar(content: string, typeHint: YamlTypeHint, proc scalar(content: string, typeHint: YamlTypeHint,
tag: TagId = tagQuestionMark, anchor: AnchorId = anchorNone): tag: TagId = tagQuestionMark, anchor: AnchorId = anchorNone):
YamlParserEvent = YamlStreamEvent =
new(result)
result.kind = yamlScalar result.kind = yamlScalar
result.scalarAnchor = anchor result.scalarAnchor = anchor
result.scalarTag = tag result.scalarTag = tag
@ -23,38 +20,33 @@ proc scalar(content: string, typeHint: YamlTypeHint,
proc scalar(content: string, proc scalar(content: string,
tag: TagId = tagQuestionMark, anchor: AnchorId = anchorNone): tag: TagId = tagQuestionMark, anchor: AnchorId = anchorNone):
YamlParserEvent = YamlStreamEvent =
result = scalar(content, yTypeUnknown, tag, anchor) result = scalar(content, yTypeUnknown, tag, anchor)
proc startSequence(tag: TagId = tagQuestionMark, proc startSequence(tag: TagId = tagQuestionMark,
anchor: AnchorId = anchorNone): anchor: AnchorId = anchorNone):
YamlParserEvent = YamlStreamEvent =
new(result)
result.kind = yamlStartSequence result.kind = yamlStartSequence
result.objAnchor = anchor result.objAnchor = anchor
result.objTag = tag result.objTag = tag
proc endSequence(): YamlParserEvent = proc endSequence(): YamlStreamEvent =
new(result)
result.kind = yamlEndSequence result.kind = yamlEndSequence
proc startMap(tag: TagId = tagQuestionMark, anchor: AnchorId = anchorNone): proc startMap(tag: TagId = tagQuestionMark, anchor: AnchorId = anchorNone):
YamlParserEvent = YamlStreamEvent =
new(result)
result.kind = yamlStartMap result.kind = yamlStartMap
result.objAnchor = anchor result.objAnchor = anchor
result.objTag = tag result.objTag = tag
proc endMap(): YamlParserEvent = proc endMap(): YamlStreamEvent =
new(result)
result.kind = yamlEndMap result.kind = yamlEndMap
proc alias(target: AnchorId): YamlParserEvent = proc alias(target: AnchorId): YamlStreamEvent =
new(result)
result.kind = yamlAlias result.kind = yamlAlias
result.aliasTarget = target result.aliasTarget = target
proc printDifference(expected, actual: YamlParserEvent) = proc printDifference(expected, actual: YamlStreamEvent) =
if expected.kind != actual.kind: if expected.kind != actual.kind:
echo "expected " & $expected.kind & ", got " & $actual.kind echo "expected " & $expected.kind & ", got " & $actual.kind
if actual.kind == yamlError: if actual.kind == yamlError:
@ -106,7 +98,7 @@ proc printDifference(expected, actual: YamlParserEvent) =
else: else:
echo "Unknown difference in event kind " & $expected.kind echo "Unknown difference in event kind " & $expected.kind
template ensure(input: string, expected: varargs[YamlParserEvent]) {.dirty.} = template ensure(input: string, expected: varargs[YamlStreamEvent]) {.dirty.} =
var var
i = 0 i = 0
events = parser.parse(newStringStream(input)) events = parser.parse(newStringStream(input))

1
test/tests.nim Normal file
View File

@ -0,0 +1 @@
import lexing, parsing