From 58ef5da17bc197386e8406f7e888eeeed1258a0b Mon Sep 17 00:00:00 2001 From: Felix Krause Date: Tue, 14 Feb 2017 19:06:41 +0100 Subject: [PATCH] Made token output conform to yaml test suite * Added -d:yamlScalarRepInd switch * Modified `$` on stream items to use yaml test suite format --- config.nims | 1 + server/server.nim | 3 +- yaml/parser.nim | 27 +++++++++++---- yaml/private/lex.nim | 11 ++++++ yaml/stream.nim | 80 ++++++++++++++++++++++++++++++++------------ 5 files changed, 93 insertions(+), 29 deletions(-) diff --git a/config.nims b/config.nims index bf83d95..44d1709 100644 --- a/config.nims +++ b/config.nims @@ -72,6 +72,7 @@ task clean, "Remove all generated files": task server, "Compile server daemon": --d:release + --d:yamlScalarRepInd setCommand "c", "server/server" task testSuiteEvents, "Compile the testSuiteEvents tool": diff --git a/server/server.nim b/server/server.nim index 7927869..c353a81 100644 --- a/server/server.nim +++ b/server/server.nim @@ -32,10 +32,11 @@ routes: of "block": style = psBlockOnly of "tokens": var - output = "" + output = "+STR\n" parser = newYamlParser() events = parser.parse(newStringStream(@"input")) for event in events: output.add($event & "\n") + output &= "-STR" resultNode["code"] = %0 resultNode["output"] = %output msg = resultNode.pretty diff --git a/yaml/parser.nim b/yaml/parser.nim index 9a316c6..2855390 100644 --- a/yaml/parser.nim +++ b/yaml/parser.nim @@ -143,7 +143,10 @@ proc initLevel(k: FastParseLevelKind): FastParseLevel {.raises: [], inline.} = FastParseLevel(kind: k, indentation: UnknownIndentation) proc emptyScalar(c: ParserContext): YamlStreamEvent {.raises: [], inline.} = - result = scalarEvent("", c.tag, c.anchor) + when defined(yamlScalarRepInd): + result = scalarEvent("", c.tag, c.anchor, srPlain) + else: + result = scalarEvent("", c.tag, c.anchor) c.tag = yTagQuestionMark c.anchor = yAnchorNone @@ -244,6 +247,12 @@ proc handlePossibleMapStart(c: ParserContext, e: var YamlStreamEvent, result = true c.level.indentation = c.lex.indentation +template implicitScalar(): YamlStreamEvent = + when defined(yamlScalarRepInd): + scalarEvent("", yTagQuestionMark, yAnchorNone, srPlain) + else: + scalarEvent("", yTagQuestionMark, yAnchorNone) + proc handleMapKeyIndicator(c: ParserContext, e: var YamlStreamEvent): bool = result = false case c.level.kind @@ -255,7 +264,7 @@ proc handleMapKeyIndicator(c: ParserContext, e: var YamlStreamEvent): bool = raise c.generateError("Invalid p.indentation of map key indicator " & "(expected" & $c.level.indentation & ", got " & $c.lex.indentation & ")") - e = scalarEvent("", yTagQuestionMark, yAnchorNone) + e = implicitScalar() result = true c.level.kind = fplMapKey c.ancestry.add(c.level) @@ -456,7 +465,7 @@ proc handleMapValueIndicator(c: ParserContext, e: var YamlStreamEvent): bool = of fplMapKey: if c.level.indentation != c.lex.indentation: raise c.generateError("Invalid p.indentation of map key indicator") - e = scalarEvent("", yTagQuestionMark, yAnchorNone) + e = implicitScalar() result = true c.level.kind = fplMapValue c.ancestry.add(c.level) @@ -640,6 +649,12 @@ parserState blockObjectStart: parserState scalarEnd: if c.tag == yTagQuestionMark: c.tag = yTagExclamationMark c.currentScalar(e) + when defined(yamlScalarRepInd): + case c.lex.scalarKind + of skSingleQuoted: e.scalarRep = srSingleQuoted + of skDoubleQuoted: e.scalarRep = srDoubleQuoted + of skLiteral: e.scalarRep = srLiteral + of skFolded: e.scalarRep = srFolded result = true state = objectEnd stored = blockAfterObject @@ -671,7 +686,7 @@ parserState blockAfterObject: e = c.objectStart(yamlStartMap) result = true of fplMapKey: - e = scalarEvent("", yTagQuestionMark, yAnchorNone) + e = implicitScalar() result = true c.level.kind = fplMapValue c.ancestry.add(c.level) @@ -771,7 +786,7 @@ parserState closeMoreIndentedLevels: state = stored parserState emitEmptyScalar: - e = scalarEvent("", yTagQuestionMark, yAnchorNone) + e = implicitScalar() result = true state = stored @@ -989,7 +1004,7 @@ parserState flowAfterObject: case c.level.kind of fplSequence: discard of fplMapValue: - e = scalarEvent("", yTagQuestionMark, yAnchorNone) + e = implicitScalar() result = true c.level.kind = fplMapKey c.explicitFlowKey = false diff --git a/yaml/private/lex.nim b/yaml/private/lex.nim index fb3e61a..d1c1431 100644 --- a/yaml/private/lex.nim +++ b/yaml/private/lex.nim @@ -9,6 +9,10 @@ when defined(yamlDebug): import terminal export terminal +when defined(yamlScalarRepInd): + type ScalarKind* = enum + skSingleQuoted, skDoubleQuoted, skLiteral, skFolded + type StringSource* = object src: string @@ -30,6 +34,9 @@ type indentation*: int # ltTagHandle shorthandEnd*: int + when defined(yamlScalarRepInd): + # ltQuotedScalar, ltBlockScalarHeader + scalarKind*: ScalarKind # may be modified from outside; will be consumed at plain scalar starts newlines*: int @@ -585,6 +592,7 @@ proc processQuotedWhitespace[T](lex: YamlLexer, newlines: var int) = proc singleQuotedScalar[T](lex: YamlLexer) = debug("lex: singleQuotedScalar") startToken[T](lex) + when defined(yamlScalarRepInd): lex.scalarKind = skSingleQuoted lex.advance(T) while true: case lex.c @@ -627,6 +635,7 @@ proc unicodeSequence[T](lex: YamlLexer, length: int) = proc doubleQuotedScalar[T](lex: YamlLexer) = debug("lex: doubleQuotedScalar") startToken[T](lex) + when defined(yamlScalarRepInd): lex.scalarKind = skDoubleQuoted lex.advance(T) while true: case lex.c @@ -778,6 +787,8 @@ proc blockScalarHeader[T](lex: YamlLexer): bool = lex.chomp = ctClip lex.blockScalarIndent = UnknownIndentation lex.folded = lex.c == '>' + when defined(yamlScalarRepInd): + lex.scalarKind = if lex.folded: skFolded else: skLiteral startToken[T](lex) while true: lex.advance(T) diff --git a/yaml/stream.nim b/yaml/stream.nim index ec2441f..c45faf0 100644 --- a/yaml/stream.nim +++ b/yaml/stream.nim @@ -15,6 +15,10 @@ import hashes import private/internal, taglib +when defined(yamlScalarRepInd): + type ScalarRepresentationIndicator* = enum + srPlain, srSingleQuoted, srDoubleQuoted, srLiteral, srFolded + type AnchorId* = distinct int ## \ ## An ``AnchorId`` identifies an anchor in the current document. It @@ -52,6 +56,8 @@ type scalarAnchor* : AnchorId scalarTag* : TagId scalarContent*: string # may not be nil (but empty) + when defined(yamlScalarRepInd): + scalarRep* : ScalarRepresentationIndicator of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard of yamlAlias: aliasTarget* : AnchorId @@ -97,8 +103,7 @@ proc noLastContext(s: YamlStream, line, column: var int, proc basicInit*(s: YamlStream, lastTokenContextImpl: proc(s: YamlStream, line, column: var int, lineContent: var string): bool - {.raises: [].} = noLastContext) - {.raises: [].} = + {.raises: [].} = noLastContext) {.raises: [].} = ## initialize basic values of the YamlStream. Call this in your constructor ## if you subclass YamlStream. s.peeked = false @@ -240,25 +245,46 @@ proc `==`*(left: YamlStreamEvent, right: YamlStreamEvent): bool {.raises: [].} = left.scalarContent == right.scalarContent of yamlAlias: result = left.aliasTarget == right.aliasTarget +proc renderAttrs(tag: TagId, anchor: AnchorId): string = + result = "" + case tag + of yTagQuestionmark: discard + of yTagExclamationmark: result &= " !" + else: result &= " !<" & $tag & ">" + if anchor != yAnchorNone: result &= " &" & $anchor + +proc yamlEscape(s: string): string = + result = "" + for c in s: + case c + of '\l': result.add("\\n") + of '\c': result.add("\\c") + of '\\': result.add("\\\\") + else: result.add(c) + proc `$`*(event: YamlStreamEvent): string {.raises: [].} = - ## outputs a human-readable string describing the given event + ## outputs a human-readable string describing the given event. + ## This string is compatible to the format used in the yaml test suite. result = $event.kind & '(' case event.kind - of yamlEndMap, yamlEndSeq, yamlStartDoc, yamlEndDoc: discard - of yamlStartMap: - result &= "tag=" & $event.mapTag - if event.mapAnchor != yAnchorNone: result &= ", anchor=" & $event.mapAnchor - of yamlStartSeq: - result &= "tag=" & $event.seqTag - if event.seqAnchor != yAnchorNone: result &= ", anchor=" & $event.seqAnchor + of yamlEndMap: result = "-MAP" + of yamlEndSeq: result = "-SEQ" + of yamlStartDoc: result = "+DOC" + of yamlEndDoc: result = "-DOC" + of yamlStartMap: result = "+MAP" & renderAttrs(event.mapTag, event.mapAnchor) + of yamlStartSeq: result = "+SEQ" & renderAttrs(event.seqTag, event.seqAnchor) of yamlScalar: - result &= "tag=" & $event.scalarTag - if event.scalarAnchor != yAnchorNone: - result &= ", anchor=" & $event.scalarAnchor - result &= ", content=\"" & event.scalarContent & '\"' - of yamlAlias: - result &= "aliasTarget=" & $event.aliasTarget - result &= ")" + result = "=VAL" & renderAttrs(event.scalarTag, event.scalarAnchor) + when defined(yamlScalarRepInd): + case event.scalarRep + of srPlain: result &= " :" + of srSingleQuoted: result &= " \'" + of srDoubleQuoted: result &= " \"" + of srLiteral: result &= " |" + of srFolded: result &= " >" + else: result &= " :" + result &= yamlEscape(event.scalarContent) + of yamlAlias: result = "=ALI *" & $event.aliasTarget proc tag*(event: YamlStreamEvent): TagId {.raises: [FieldError].} = ## returns the tag of the given event @@ -294,11 +320,21 @@ proc endSeqEvent*(): YamlStreamEvent {.inline, raises: [].} = ## creates a new event that marks the end of a YAML sequence result = YamlStreamEvent(kind: yamlEndSeq) -proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, - anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} = - ## creates a new event that represents a YAML scalar - result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag, - scalarAnchor: anchor, scalarContent: content) +when defined(yamlScalarRepInd): + proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, + anchor: AnchorId = yAnchorNone, + scalarRep: ScalarRepresentationIndicator = srPlain): + YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that represents a YAML scalar + result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag, + scalarAnchor: anchor, scalarContent: content, + scalarRep: scalarRep) +else: + proc scalarEvent*(content: string = "", tag: TagId = yTagQuestionMark, + anchor: AnchorId = yAnchorNone): YamlStreamEvent {.inline, raises: [].} = + ## creates a new event that represents a YAML scalar + result = YamlStreamEvent(kind: yamlScalar, scalarTag: tag, + scalarAnchor: anchor, scalarContent: content) proc aliasEvent*(anchor: AnchorId): YamlStreamEvent {.inline, raises: [].} = ## creates a new event that represents a YAML alias