status-im/nim-json-serialization
2
0
Fork 0
mirror of https://github.com/status-im/nim-json-serialization.git synced 2025-02-22 06:48:24 +00:00
Code Issues Projects Releases Wiki Activity

Use the improved InputStream API

Browse Source
This commit is contained in:
Zahary Karadjov 2020-04-10 16:46:12 +03:00
parent bdddff5037
commit 16931f4fa3
No known key found for this signature in database
GPG Key ID: C8936F8A3073D609
4 changed files with 59 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
json_serialization/lexer.nim
View File

@ -41,7 +41,7 @@ type
errNonPortableInt = "number is outside the range of portable values" errNonPortableInt = "number is outside the range of portable values"
JsonLexer* = object JsonLexer* = object
stream: ref AsciiStream stream: AsciiInputStream
mode*: JsonMode mode*: JsonMode
line*: int line*: int
@ -72,12 +72,12 @@ proc isDigit(c: char): bool {.inline.} =
return (c >= '0' and c <= '9') return (c >= '0' and c <= '9')
proc col*(lexer: JsonLexer): int = proc col*(lexer: JsonLexer): int =
lexer.stream[].pos - lexer.lineStartPos lexer.stream.pos - lexer.lineStartPos
proc tokenStartCol*(lexer: JsonLexer): int = proc tokenStartCol*(lexer: JsonLexer): int =
1 + lexer.tokenStart - lexer.lineStartPos 1 + lexer.tokenStart - lexer.lineStartPos
proc init*(T: type JsonLexer, stream: ref AsciiStream, mode = defaultJsonMode): T = proc init*(T: type JsonLexer, stream: AsciiInputStream, mode = defaultJsonMode): T =
T(stream: stream, T(stream: stream,
mode: mode, mode: mode,
line: 1, line: 1,
@ -89,9 +89,8 @@ proc init*(T: type JsonLexer, stream: ref AsciiStream, mode = defaultJsonMode):
floatVal: 0'f, floatVal: 0'f,
strVal: "") strVal: "")
proc init*(T: type JsonLexer, stream: ref ByteStream, mode = defaultJsonMode): auto = proc init*(T: type JsonLexer, stream: InputStream, mode = defaultJsonMode): T =
type AS = ref AsciiStream init(JsonLexer, AsciiInputStream(stream), mode)
init(JsonLexer, AS(stream), mode)
template error(error: JsonErrorKind) {.dirty.} = template error(error: JsonErrorKind) {.dirty.} =
lexer.err = error lexer.err = error
@ -99,11 +98,11 @@ template error(error: JsonErrorKind) {.dirty.} =
return return
template checkForUnexpectedEof {.dirty.} = template checkForUnexpectedEof {.dirty.} =
if lexer.stream[].eof: error errUnexpectedEof if lexer.stream.eof: error errUnexpectedEof
template requireNextChar(): char = template requireNextChar(): char =
checkForUnexpectedEof() checkForUnexpectedEof()
lexer.stream[].read() lexer.stream.read()
template checkForNonPortableInt(val: uint64) = template checkForNonPortableInt(val: uint64) =
if lexer.mode == Portable and val > uint64(maxPortableInt): if lexer.mode == Portable and val > uint64(maxPortableInt):
@ -118,9 +117,9 @@ proc scanHexRune(lexer: var JsonLexer): int =
proc scanString(lexer: var JsonLexer) = proc scanString(lexer: var JsonLexer) =
lexer.tok = tkString lexer.tok = tkString
lexer.strVal.setLen 0 lexer.strVal.setLen 0
lexer.tokenStart = lexer.stream[].pos lexer.tokenStart = lexer.stream.pos
advance lexer.stream[] advance lexer.stream
while true: while true:
var c = requireNextChar() var c = requireNextChar()
@ -167,32 +166,32 @@ proc scanString(lexer: var JsonLexer) =
lexer.strVal.add c lexer.strVal.add c
proc handleLF(lexer: var JsonLexer) {.inline.} = proc handleLF(lexer: var JsonLexer) {.inline.} =
advance lexer.stream[] advance lexer.stream
lexer.line += 1 lexer.line += 1
lexer.lineStartPos = lexer.stream[].pos lexer.lineStartPos = lexer.stream.pos
proc skipWhitespace(lexer: var JsonLexer) = proc skipWhitespace(lexer: var JsonLexer) =
template handleCR = template handleCR =
# Beware: this is a template, because the return # Beware: this is a template, because the return
# statement has to exit `skipWhitespace`. # statement has to exit `skipWhitespace`.
advance lexer.stream[] advance lexer.stream
if lexer.stream[].eof: return if lexer.stream.eof: return
if lexer.stream[].peek() == '\n': advance lexer.stream[] if lexer.stream.peek() == '\n': advance lexer.stream
lexer.line += 1 lexer.line += 1
lexer.lineStartPos = lexer.stream[].pos lexer.lineStartPos = lexer.stream.pos
while true: while true:
if lexer.stream[].eof: return if lexer.stream.eof: return
case lexer.stream[].peek() case lexer.stream.peek()
of '/': of '/':
advance lexer.stream[] advance lexer.stream
checkForUnexpectedEof() checkForUnexpectedEof()
case lexer.stream[].peek() case lexer.stream.peek()
of '/': of '/':
while true: while true:
advance lexer.stream[] advance lexer.stream
if lexer.stream[].eof: return if lexer.stream.eof: return
case lexer.stream[].peek() case lexer.stream.peek()
of '\r': of '\r':
handleCR() handleCR()
of '\n': of '\n':
@ -201,25 +200,25 @@ proc skipWhitespace(lexer: var JsonLexer) =
discard discard
of '*': of '*':
while true: while true:
advance lexer.stream[] advance lexer.stream
if lexer.stream[].eof: return if lexer.stream.eof: return
case lexer.stream[].peek() case lexer.stream.peek()
of '\r': of '\r':
handleCR() handleCR()
of '\n': of '\n':
lexer.handleLF() lexer.handleLF()
of '*': of '*':
advance lexer.stream[] advance lexer.stream
checkForUnexpectedEof() checkForUnexpectedEof()
if lexer.stream[].peek() == '/': if lexer.stream.peek() == '/':
advance lexer.stream[] advance lexer.stream
return return
else: else:
discard discard
else: else:
error errCommentExpected error errCommentExpected
of ' ', '\t': of ' ', '\t':
advance lexer.stream[] advance lexer.stream
of '\r': of '\r':
handleCR() handleCR()
of '\n': of '\n':
@ -228,32 +227,32 @@ proc skipWhitespace(lexer: var JsonLexer) =
break break
template requireMoreNumberChars(elseClause) = template requireMoreNumberChars(elseClause) =
if lexer.stream[].eof: if lexer.stream.eof:
elseClause elseClause
error errNumberExpected error errNumberExpected
template eatDigitAndPeek: char = template eatDigitAndPeek: char =
advance lexer.stream[] advance lexer.stream
if lexer.stream[].eof: return if lexer.stream.eof: return
lexer.stream[].peek() lexer.stream.peek()
proc scanSign(lexer: var JsonLexer): int = proc scanSign(lexer: var JsonLexer): int =
# Returns +1 or -1 # Returns +1 or -1
# If a sign character is present, it must be followed # If a sign character is present, it must be followed
# by more characters representing the number. If this # by more characters representing the number. If this
# is not the case, the return value will be 0. # is not the case, the return value will be 0.
let c = lexer.stream[].peek() let c = lexer.stream.peek()
if c == '-': if c == '-':
requireMoreNumberChars: result = 0 requireMoreNumberChars: result = 0
advance lexer.stream[] advance lexer.stream
return -1 return -1
elif c == '+': elif c == '+':
requireMoreNumberChars: result = 0 requireMoreNumberChars: result = 0
advance lexer.stream[] advance lexer.stream
return 1 return 1
proc scanInt(lexer: var JsonLexer): uint64 = proc scanInt(lexer: var JsonLexer): uint64 =
var c = lexer.stream[].peek() var c = lexer.stream.peek()
result = uint64(ord(c) - ord('0')) result = uint64(ord(c) - ord('0'))
c = eatDigitAndPeek() c = eatDigitAndPeek()
@ -264,21 +263,21 @@ proc scanInt(lexer: var JsonLexer): uint64 =
proc scanNumber(lexer: var JsonLexer) = proc scanNumber(lexer: var JsonLexer) =
var sign = lexer.scanSign() var sign = lexer.scanSign()
if sign == 0: return if sign == 0: return
var c = lexer.stream[].peek() var c = lexer.stream.peek()
if c == '.': if c == '.':
advance lexer.stream[] advance lexer.stream
requireMoreNumberChars: discard requireMoreNumberChars: discard
lexer.tok = tkFloat lexer.tok = tkFloat
c = lexer.stream[].peek() c = lexer.stream.peek()
elif c.isDigit: elif c.isDigit:
lexer.tok = if sign > 0: tkInt lexer.tok = if sign > 0: tkInt
else: tkNegativeInt else: tkNegativeInt
let scannedValue = lexer.scanInt() let scannedValue = lexer.scanInt()
checkForNonPortableInt scannedValue checkForNonPortableInt scannedValue
lexer.absIntVal = scannedValue lexer.absIntVal = scannedValue
if lexer.stream[].eof: return if lexer.stream.eof: return
c = lexer.stream[].peek() c = lexer.stream.peek()
if c == '.': if c == '.':
lexer.tok = tkFloat lexer.tok = tkFloat
lexer.floatVal = float(lexer.absIntVal) * float(sign) lexer.floatVal = float(lexer.absIntVal) * float(sign)
@ -293,11 +292,11 @@ proc scanNumber(lexer: var JsonLexer) =
c = eatDigitAndPeek() c = eatDigitAndPeek()
if c in {'E', 'e'}: if c in {'E', 'e'}:
advance lexer.stream[] advance lexer.stream
requireMoreNumberChars: discard requireMoreNumberChars: discard
let sign = lexer.scanSign() let sign = lexer.scanSign()
if sign == 0: return if sign == 0: return
if not isDigit lexer.stream[].peek(): if not isDigit lexer.stream.peek():
error errNumberExpected error errNumberExpected
let exponent = lexer.scanInt() let exponent = lexer.scanInt()
@ -312,7 +311,7 @@ proc scanNumber(lexer: var JsonLexer) =
proc scanIdentifier(lexer: var JsonLexer, proc scanIdentifier(lexer: var JsonLexer,
expectedIdent: string, expectedTok: TokKind) = expectedIdent: string, expectedTok: TokKind) =
for c in expectedIdent: for c in expectedIdent:
if c != lexer.stream[].read(): if c != lexer.stream.read():
lexer.tok = tkError lexer.tok = tkError
return return
lexer.tok = expectedTok lexer.tok = expectedTok
@ -320,33 +319,33 @@ proc scanIdentifier(lexer: var JsonLexer,
proc next*(lexer: var JsonLexer) = proc next*(lexer: var JsonLexer) =
lexer.skipWhitespace() lexer.skipWhitespace()
if lexer.stream[].eof: if lexer.stream.eof:
lexer.tok = tkEof lexer.tok = tkEof
return return
let c = lexer.stream[].peek() let c = lexer.stream.peek()
case c case c
of '+', '-', '.', '0'..'9': of '+', '-', '.', '0'..'9':
lexer.scanNumber() lexer.scanNumber()
of '"': of '"':
lexer.scanString() lexer.scanString()
of '[': of '[':
advance lexer.stream[] advance lexer.stream
lexer.tok = tkBracketLe lexer.tok = tkBracketLe
of '{': of '{':
advance lexer.stream[] advance lexer.stream
lexer.tok = tkCurlyLe lexer.tok = tkCurlyLe
of ']': of ']':
advance lexer.stream[] advance lexer.stream
lexer.tok = tkBracketRi lexer.tok = tkBracketRi
of '}': of '}':
advance lexer.stream[] advance lexer.stream
lexer.tok = tkCurlyRi lexer.tok = tkCurlyRi
of ',': of ',':
advance lexer.stream[] advance lexer.stream
lexer.tok = tkComma lexer.tok = tkComma
of ':': of ':':
advance lexer.stream[] advance lexer.stream
lexer.tok = tkColon lexer.tok = tkColon
of '\0': of '\0':
lexer.tok = tkEof lexer.tok = tkEof
@ -354,6 +353,6 @@ proc next*(lexer: var JsonLexer) =
of 't': lexer.scanIdentifier("true", tkTrue) of 't': lexer.scanIdentifier("true", tkTrue)
of 'f': lexer.scanIdentifier("false", tkFalse) of 'f': lexer.scanIdentifier("false", tkFalse)
else: else:
advance lexer.stream[] advance lexer.stream
lexer.tok = tkError lexer.tok = tkError

6
json_serialization/reader.nim
View File

@ -61,8 +61,8 @@ method formatMsg*(err: ref GenericJsonReaderError, filename: string): string =
method formatMsg*(err: ref IntOverflowError, filename: string): string = method formatMsg*(err: ref IntOverflowError, filename: string): string =
fmt"{filename}({err.line}, {err.col}) The value '{err.valueStr}' is outside of the allowed range" fmt"{filename}({err.line}, {err.col}) The value '{err.valueStr}' is outside of the allowed range"
template init*(T: type JsonReader, stream: ByteStreamVar, mode = defaultJsonMode): auto = template init*(T: type JsonReader, stream: InputStream, mode = defaultJsonMode): auto =
init JsonReader, AsciiStreamVar(stream), mode init JsonReader, AsciiInputStream(stream), mode
proc assignLineNumber*(ex: ref JsonReaderError, r: JsonReader) = proc assignLineNumber*(ex: ref JsonReaderError, r: JsonReader) =
ex.line = r.lexer.line ex.line = r.lexer.line
@ -100,7 +100,7 @@ proc handleReadException*(r: JsonReader,
ex.innerException = err ex.innerException = err
raise ex raise ex
proc init*(T: type JsonReader, stream: AsciiStreamVar, mode = defaultJsonMode): T = proc init*(T: type JsonReader, stream: AsciiInputStream, mode = defaultJsonMode): T =
result.lexer = JsonLexer.init(stream, mode) result.lexer = JsonLexer.init(stream, mode)
result.lexer.next() result.lexer.next()

2
json_serialization/writer.nim
View File

@ -199,7 +199,7 @@ proc writeValue*(w: var JsonWriter, value: auto) =
proc toJson*(v: auto, pretty = false, typeAnnotations = false): string = proc toJson*(v: auto, pretty = false, typeAnnotations = false): string =
mixin writeValue mixin writeValue
var s = init OutputStream var s = memoryOutput()
var w = JsonWriter.init(s, pretty, typeAnnotations) var w = JsonWriter.init(s, pretty, typeAnnotations)
w.writeValue v w.writeValue v
return s.getOutput(string) return s.getOutput(string)

2
tests/test_lexer.nim
View File

@ -10,7 +10,7 @@ template expectedToken(token: TokKind, additionalCheck = true) {.dirty.} =
template lexerTest(name, input: string, expectations) {.dirty.} = template lexerTest(name, input: string, expectations) {.dirty.} =
test name: test name:
var stream = memoryStream(dedent(input)) var stream = memoryInput(dedent(input))
var lexer = JsonLexer.init stream var lexer = JsonLexer.init stream
expectations expectations