From dcfec8aaa345c0a5e6c97d08558d1efa1ebb7e6a Mon Sep 17 00:00:00 2001 From: munna0908 Date: Fri, 16 May 2025 18:24:50 +0530 Subject: [PATCH 01/20] add cbor types --- serde.nim | 2 + serde/cbor/serializer.nim | 221 ++++++++++++++++++++++++++++++ serde/json.nim | 6 +- serde/json/deserializer.nim | 8 +- serde/json/parser.nim | 4 +- serde/json/serializer.nim | 6 +- serde/utils/cbor.nim | 42 ++++++ serde/{json => utils}/errors.nim | 2 + serde/{json => utils}/pragmas.nim | 0 serde/{json => utils}/stdjson.nim | 2 +- serde/{json => utils}/types.nim | 1 + 11 files changed, 281 insertions(+), 13 deletions(-) create mode 100644 serde/cbor/serializer.nim create mode 100644 serde/utils/cbor.nim rename serde/{json => utils}/errors.nim (96%) rename serde/{json => utils}/pragmas.nim (100%) rename serde/{json => utils}/stdjson.nim (52%) rename serde/{json => utils}/types.nim (94%) diff --git a/serde.nim b/serde.nim index db31316..cb6b1db 100644 --- a/serde.nim +++ b/serde.nim @@ -1,3 +1,5 @@ import ./serde/json +import ./serde/cbor export json +export cbor diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim new file mode 100644 index 0000000..9b39e60 --- /dev/null +++ b/serde/cbor/serializer.nim @@ -0,0 +1,221 @@ +{.push checks: off.} + +import ../utils/cbor + +func initialByte(major, minor: Natural): uint8 {.inline.} = + uint8((major shl 5) or (minor and 0b11111)) + +proc writeInitial[T: SomeInteger](str: Stream; m: uint8; n: T) = + ## Write the initial integer of a CBOR item. + let m = m shl 5 + when T is byte: + if n < 24: + str.write(m or n.uint8) + else: + str.write(m or 24'u8) + str.write(n) + else: + if n < 24: + str.write(m or n.uint8) + elif uint64(n) <= uint64(uint8.high): + str.write(m or 24'u8) + str.write(n.uint8) + elif uint64(n) <= uint64(uint16.high): + str.write(m or 25'u8) + str.write((uint8)n shr 8) + str.write((uint8)n) + elif uint64(n) <= uint64(uint32.high): + str.write(m or 26'u8) + for i in countdown(24, 8, 8): + {.unroll.} + str.write((uint8)n shr i) + str.write((uint8)n) + else: + str.write(m or 27'u8) + for i in countdown(56, 8, 8): + {.unroll.} + str.write((uint8)n shr i) + str.write((uint8)n) +{.pop.} + +proc writeCborArrayLen*(str: Stream; len: Natural) = + ## Write a marker to the stream that initiates an array of ``len`` items. + str.writeInitial(4, len) + +proc writeCborIndefiniteArrayLen*(str: Stream) = + ## Write a marker to the stream that initiates an array of indefinite length. + ## Indefinite length arrays are composed of an indefinite amount of arrays + ## of definite lengths. + str.write(initialByte(4, 31)) + +proc writeCborMapLen*(str: Stream; len: Natural) = + ## Write a marker to the stream that initiates an map of ``len`` pairs. + str.writeInitial(5, len) + +proc writeCborIndefiniteMapLen*(str: Stream) = + ## Write a marker to the stream that initiates a map of indefinite length. + ## Indefinite length maps are composed of an indefinite amount of maps + ## of definite length. + str.write(initialByte(5, 31)) + +proc writeCborBreak*(str: Stream) = + ## Write a marker to the stream that ends an indefinite array or map. + str.write(initialByte(7, 31)) + +proc writeCborTag*(str: Stream; tag: Natural) {.inline.} = + ## Write a tag for the next CBOR item to a binary stream. + str.writeInitial(6, tag) + +proc writeCbor*(str: Stream; buf: pointer; len: int) = + ## Write a raw buffer to a CBOR `Stream`. + str.writeInitial(BytesMajor, len) + if len > 0: str.writeData(buf, len) + +proc writeCbor*[T](str: Stream; v: T) = + ## Write the CBOR binary representation of a `T` to a `Stream`. + ## The behavior of this procedure can be extended or overriden + ## by defining `proc writeCborHook(str: Stream; v: T)` for specific + ## types `T`. + when T is CborNode: + if v.tag.isSome: + str.writeCborTag(v.tag.get) + case v.kind: + of cborUnsigned: + str.writeCbor(v.uint) + of cborNegative: + str.writeCbor(v.int) + of cborBytes: + str.writeInitial(cborBytes.uint8, v.bytes.len) + for b in v.bytes.items: + str.write(b) + of cborText: + str.writeInitial(cborText.uint8, v.text.len) + str.write(v.text) + of cborArray: + str.writeInitial(4, v.seq.len) + for e in v.seq: + str.writeCbor(e) + of cborMap: + assert(v.isSorted, "refusing to write unsorted map to stream") + str.writeInitial(5, v.map.len) + for k, f in v.map.pairs: + str.writeCbor(k) + str.writeCbor(f) + of cborTag: + discard + of cborSimple: + if v.simple > 31'u or v.simple == 24: + str.write(initialByte(cborSimple.uint8, 24)) + str.write(v.simple) + else: + str.write(initialByte(cborSimple.uint8, v.simple)) + of cborFloat: + str.writeCbor(v.float) + of cborRaw: + str.write(v.raw) + elif compiles(writeCborHook(str, v)): + writeCborHook(str, v) + elif T is SomeUnsignedInt: + str.writeInitial(0, v) + elif T is SomeSignedInt: + if v < 0: + # Major type 1 + str.writeInitial(1, -1 - v) + else: + # Major type 0 + str.writeInitial(0, v) + elif T is seq[byte]: + str.writeInitial(BytesMajor, v.len) + if v.len > 0: + str.writeData(unsafeAddr v[0], v.len) + elif T is openArray[char | uint8 | int8]: + str.writeInitial(BytesMajor, v.len) + if v.len > 0: + str.writeData(unsafeAddr v[0], v.len) + elif T is string: + str.writeInitial(TextMajor, v.len) + str.write(v) + elif T is array | seq: + str.writeInitial(4, v.len) + for e in v.items: + writeCbor(str, e) + elif T is tuple: + str.writeInitial(4, T.tupleLen) + for f in v.fields: str.writeCbor(f) + elif T is ptr | ref: + if system.`==`(v, nil): + # Major type 7 + str.write(Null) + else: writeCbor(str, v[]) + elif T is object: + var n: uint + for _, _ in v.fieldPairs: + inc n + str.writeInitial(5, n) + for k, f in v.fieldPairs: + str.writeCbor(k) + str.writeCbor(f) + elif T is bool: + str.write(initialByte(7, (if v: 21 else: 20))) + elif T is SomeFloat: + case v.classify + of fcNormal, fcSubnormal: + let single = v.float32 + if single.float64 == v.float64: + if single.isHalfPrecise: + let half = floatHalf(single) + str.write(initialByte(7, 25)) + when system.cpuEndian == bigEndian: + str.write(half) + else: + var be: uint16 + swapEndian16 be.addr, half.unsafeAddr + str.write(be) + else: + str.write initialByte(7, 26) + when system.cpuEndian == bigEndian: + str.write(single) + else: + var be: uint32 + swapEndian32 be.addr, single.unsafeAddr + str.write(be) + else: + str.write initialByte(7, 27) + when system.cpuEndian == bigEndian: + str.write(v) + else: + var be: float64 + swapEndian64 be.addr, v.unsafeAddr + str.write be + return + of fcZero: + str.write initialByte(7, 25) + str.write((char)0x00) + of fcNegZero: + str.write initialByte(7, 25) + str.write((char)0x80) + of fcInf: + str.write initialByte(7, 25) + str.write((char)0x7c) + of fcNan: + str.write initialByte(7, 25) + str.write((char)0x7e) + of fcNegInf: + str.write initialByte(7, 25) + str.write((char)0xfc) + str.write((char)0x00) + +proc writeCborArray*(str: Stream; args: varargs[CborNode, toCbor]) = + ## Encode to a CBOR array in binary form. This magic doesn't + ## always work, some arguments may need to be explicitly + ## converted with ``toCbor`` before passing. + str.writeCborArrayLen(args.len) + for x in args: + str.writeCbor(x) + +proc encode*[T](v: T): string = + ## Encode an arbitrary value to CBOR binary representation. + ## A wrapper over ``writeCbor``. + let s = newStringStream() + s.writeCbor(v) + s.data diff --git a/serde/json.nim b/serde/json.nim index 407c66c..5d61c62 100644 --- a/serde/json.nim +++ b/serde/json.nim @@ -1,9 +1,9 @@ import ./json/parser import ./json/deserializer -import ./json/stdjson -import ./json/pragmas +import ./utils/stdjson +import ./utils/pragmas import ./json/serializer -import ./json/types +import ./utils/types export parser export deserializer diff --git a/serde/json/deserializer.nim b/serde/json/deserializer.nim index 41c4106..6227e97 100644 --- a/serde/json/deserializer.nim +++ b/serde/json/deserializer.nim @@ -12,10 +12,10 @@ import pkg/questionable import pkg/questionable/results import ./parser -import ./errors -import ./stdjson -import ./pragmas -import ./types +import ../utils/errors +import ../utils/stdjson +import ../utils/pragmas +import ../utils/types import ./helpers export parser diff --git a/serde/json/parser.nim b/serde/json/parser.nim index 5f790e1..c68e79c 100644 --- a/serde/json/parser.nim +++ b/serde/json/parser.nim @@ -2,8 +2,8 @@ import std/json as stdjson import pkg/questionable/results -import ./errors -import ./types +import ../utils/errors +import ../utils/types {.push raises: [].} diff --git a/serde/json/serializer.nim b/serde/json/serializer.nim index 00fdfa2..d405a7d 100644 --- a/serde/json/serializer.nim +++ b/serde/json/serializer.nim @@ -9,9 +9,9 @@ import pkg/questionable import pkg/stew/byteutils import pkg/stint -import ./stdjson -import ./pragmas -import ./types +import ../utils/stdjson +import ../utils/pragmas +import ../utils/types export chronicles except toJson export stdjson diff --git a/serde/utils/cbor.nim b/serde/utils/cbor.nim new file mode 100644 index 0000000..ebe91d1 --- /dev/null +++ b/serde/utils/cbor.nim @@ -0,0 +1,42 @@ +import std/[tables] + +type CborNodeKind* = enum + cborUnsigned = 0, + cborNegative = 1, + cborBytes = 2, + cborText = 3, + cborArray = 4, + cborMap = 5, + cborTag = 6, + cborSimple = 7, + cborFloat, + cborRaw + + CborNode* = object + ## An abstract representation of a CBOR item. Useful for diagnostics. + tag: Option[uint64] + case kind*: CborNodeKind + of cborUnsigned: + uint*: BiggestUInt + of cborNegative: + int*: BiggestInt + of cborBytes: + bytes*: seq[byte] + of cborText: + text*: string + of cborArray: + seq*: seq[CborNode] + of cborMap: + map*: OrderedTable[CborNode, CborNode] + of cborTag: + discard + of cborSimple: + simple*: uint8 + of cborFloat: + float*: float64 + of cborRaw: + raw*: string + +func `==`*(x, y: CborNode): bool + +func hash*(x: CborNode): Hash \ No newline at end of file diff --git a/serde/json/errors.nim b/serde/utils/errors.nim similarity index 96% rename from serde/json/errors.nim rename to serde/utils/errors.nim index 2ee1c0d..4523c09 100644 --- a/serde/json/errors.nim +++ b/serde/utils/errors.nim @@ -13,6 +13,8 @@ proc mapErrTo*[E1: ref CatchableError, E2: SerdeError]( proc newSerdeError*(msg: string): ref SerdeError = newException(SerdeError, msg) +# proc newUnexpectedKindError* + proc newUnexpectedKindError*( expectedType: type, expectedKinds: string, json: JsonNode ): ref UnexpectedKindError = diff --git a/serde/json/pragmas.nim b/serde/utils/pragmas.nim similarity index 100% rename from serde/json/pragmas.nim rename to serde/utils/pragmas.nim diff --git a/serde/json/stdjson.nim b/serde/utils/stdjson.nim similarity index 52% rename from serde/json/stdjson.nim rename to serde/utils/stdjson.nim index 1adde16..9855bd5 100644 --- a/serde/json/stdjson.nim +++ b/serde/utils/stdjson.nim @@ -1,3 +1,3 @@ import std/json except `%`, `%*`, parseJson -export json except `%`, `%*`, parseJson +export json except `%`, `%*`, parseJson \ No newline at end of file diff --git a/serde/json/types.nim b/serde/utils/types.nim similarity index 94% rename from serde/json/types.nim rename to serde/utils/types.nim index facb3b6..c26cd15 100644 --- a/serde/json/types.nim +++ b/serde/utils/types.nim @@ -1,6 +1,7 @@ type SerdeError* = object of CatchableError JsonParseError* = object of SerdeError + CborParseError* = object of SerdeError UnexpectedKindError* = object of SerdeError SerdeMode* = enum OptOut From f1ec805ec01d9da45cc923f6e83bc548ad8153b6 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Wed, 21 May 2025 03:22:35 +0530 Subject: [PATCH 02/20] feat: add CBOR serialization and deserialization support --- .gitignore | 3 +- serde.nimble | 2 +- serde/cbor.nim | 7 + serde/cbor/deserializer.nim | 761 ++++++++++++++++++++++++++++++ serde/cbor/errors.nim | 37 ++ serde/cbor/helpers.nim | 42 ++ serde/cbor/jsonhook.nim | 42 ++ serde/cbor/serializer.nim | 152 +++++- serde/cbor/types.nim | 165 +++++++ serde/json.nim | 2 +- serde/json/deserializer.nim | 38 +- serde/json/errors.nim | 28 ++ serde/json/helpers.nim | 27 ++ serde/json/serializer.nim | 4 +- serde/{utils => json}/stdjson.nim | 0 serde/utils/cbor.nim | 42 -- serde/utils/errors.nim | 26 - tests/cbor/test.nim | 100 ++++ tests/cbor/testDeserialize.nim | 260 ++++++++++ tests/cbor/test_vector.json | 685 +++++++++++++++++++++++++++ tests/test.nim | 2 + tests/test.nimble | 1 + 22 files changed, 2322 insertions(+), 104 deletions(-) create mode 100644 serde/cbor.nim create mode 100644 serde/cbor/deserializer.nim create mode 100644 serde/cbor/errors.nim create mode 100644 serde/cbor/helpers.nim create mode 100644 serde/cbor/jsonhook.nim create mode 100644 serde/cbor/types.nim create mode 100644 serde/json/errors.nim rename serde/{utils => json}/stdjson.nim (100%) delete mode 100644 serde/utils/cbor.nim create mode 100644 tests/cbor/test.nim create mode 100644 tests/cbor/testDeserialize.nim create mode 100644 tests/cbor/test_vector.json diff --git a/.gitignore b/.gitignore index 2681317..dcd6561 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ nimble.develop nimble.paths .idea vendor/ -.vscode/ \ No newline at end of file +.vscode/ +nimbledeps \ No newline at end of file diff --git a/serde.nimble b/serde.nimble index 187b395..4d5af94 100644 --- a/serde.nimble +++ b/serde.nimble @@ -9,7 +9,7 @@ skipDirs = @["tests"] # Dependencies requires "nim >= 1.6.14" requires "chronicles >= 0.10.3 & < 0.11.0" -requires "questionable >= 0.10.13 & < 0.11.0" +requires "questionable >= 0.10.15" requires "stint" requires "stew" diff --git a/serde/cbor.nim b/serde/cbor.nim new file mode 100644 index 0000000..de12a78 --- /dev/null +++ b/serde/cbor.nim @@ -0,0 +1,7 @@ +import ./cbor/serializer +import ./cbor/deserializer +import ./cbor/jsonhook +import ./cbor/types as ctypes +import ./utils/types +import ./utils/errors +export serializer, deserializer, ctypes, types, errors, jsonhook diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim new file mode 100644 index 0000000..cdff43b --- /dev/null +++ b/serde/cbor/deserializer.nim @@ -0,0 +1,761 @@ +import std/[math, streams, options, tables, strutils, times, typetraits] +import ./types +import ./helpers +import ../utils/types +import ./errors +import pkg/questionable +import pkg/questionable/results + +export results +export types + + +func isIndefinite*(c: CborParser): bool {.inline.} = c.minor == 31 + ## Return true if the parser is positioned on an item of indefinite length. + +{.push raises: [].} +proc open*(c: var CborParser; s: Stream) = + ## Begin parsing a stream of CBOR in binary form. + ## The parser will be initialized in an EOF state, call + ## ``next`` to advance it before parsing. + c.s = s + c.kind = cborEof + c.intVal = 0 + +proc next*(c: var CborParser): ?!void = + ## Advance the parser to the initial or next event. + try: + if c.s.atEnd: + c.kind = CborEventKind.cborEof + c.intVal = 0 + else: + let + ib = c.s.readUint8 + mb = ib shr 5 + c.minor = ib and 0b11111 + case c.minor + of 0..23: + c.intVal = c.minor.uint64 + of 24: + c.intVal = c.s.readChar.uint64 + of 25: + c.intVal = c.s.readChar.uint64 + c.intVal = (c.intVal shl 8) or c.s.readChar.uint64 + of 26: + c.intVal = c.s.readChar.uint64 + for _ in 1..3: + {.unroll.} + c.intVal = (c.intVal shl 8) or c.s.readChar.uint64 + of 27: + c.intVal = c.s.readChar.uint64 + for _ in 1..7: + {.unroll.} + c.intVal = (c.intVal shl 8) or c.s.readChar.uint64 + else: + c.intVal = 0 + case mb + of PositiveMajor: + c.kind = CborEventKind.cborPositive + of NegativeMajor: + c.kind = CborEventKind.cborNegative + of BytesMajor: + c.kind = CborEventKind.cborBytes + of TextMajor: + c.kind = CborEventKind.cborText + of ArrayMajor: + c.kind = CborEventKind.cborArray + of MapMajor: + c.kind = CborEventKind.cborMap + of TagMajor: + c.kind = CborEventKind.cborTag + of SimpleMajor: + if c.minor in {25, 26, 27}: + c.kind = CborEventKind.cborFloat + elif c.isIndefinite: + c.kind = CborEventKind.cborBreak + else: + c.kind = CborEventKind.cborSimple + else: + return failure(newCborError("unhandled major type " & $mb)) + success() + except IOError as e: + return failure(e) + except OSError as e: + return failure(e) + +proc nextUInt*(c: var CborParser): ?!BiggestUInt = + ## Parse the integer value that the parser is positioned on. + if c.kind != CborEventKind.cborPositive: + return failure(newCborError("Expected positive integer, got " & $c.kind)) + let val = c.intVal.BiggestUInt + let nextRes = c.next() + if nextRes.isFailure: + return failure(nextRes.error) + return success(val) + + +proc nextInt*(c: var CborParser): ?!BiggestInt = + ## Parse the integer value that the parser is positioned on. + var val: BiggestInt + case c.kind + of CborEventKind.cborPositive: + val = c.intVal.BiggestInt + of CborEventKind.cborNegative: + val = -1.BiggestInt - c.intVal.BiggestInt + else: + return failure(newCborError("Expected integer, got " & $c.kind)) + + let nextRes = c.next() + if nextRes.isFailure: + return failure(nextRes.error) + + return success(val) + +proc nextFloat*(c: var CborParser): ?!float64 = + ## Parse the float value that the parser is positioned on. + var val: float64 + if c.kind != CborEventKind.cborFloat: + return failure(newCborError("Expected float, got " & $c.kind)) + case c.minor + of 25: + val = floatSingle(c.intVal.uint16).float64 + of 26: + val = cast[float32](c.intVal).float64 + of 27: + val = cast[float64](c.intVal) + else: + discard + + let nextRes = c.next() + if nextRes.isFailure: + return failure(nextRes.error) + + return success(val) + +func bytesLen*(c: CborParser): ?!int = + ## Return the length of the byte string that the parser is positioned on. + if c.kind != CborEventKind.cborBytes: + return failure(newCborError("Expected bytes, got " & $c.kind)) + return success(c.intVal.int) + +template tryNext(c: var CborParser) = + let nextRes = c.next() + if nextRes.isFailure: + return failure(nextRes.error) + +template trySkip(c: var CborParser) = + let skipRes = c.skipNode() + if skipRes.isFailure: + return failure(skipRes.error) + +proc nextBytes*(c: var CborParser; buf: var openArray[byte]): ?!void = + ## Read the bytes that the parser is positioned on and advance. + try: + if c.kind != CborEventKind.cborBytes: + return failure(newCborError("Expected bytes, got " & $c.kind)) + if buf.len != c.intVal.int: + return failure(newCborError("Buffer length mismatch: expected " & + $c.intVal.int & ", got " & $buf.len)) + if buf.len > 0: + let n = c.s.readData(buf[0].addr, buf.len) + if n != buf.len: + return failure(newCborError("truncated read of CBOR data")) + tryNext(c) + success() + except OSError as e: + return failure(e.msg) + except IOError as e: + return failure(e.msg) + +proc nextBytes*(c: var CborParser): ?!seq[byte] = + ## Read the bytes that the parser is positioned on into a seq and advance. + var val = newSeq[byte](c.intVal.int) + let nextRes = nextBytes(c, val) + if nextRes.isFailure: + return failure(nextRes.error) + + return success(val) + +func textLen*(c: CborParser): ?!int = + ## Return the length of the text that the parser is positioned on. + if c.kind != CborEventKind.cborText: + return failure(newCborError("Expected text, got " & $c.kind)) + return success(c.intVal.int) + +proc nextText*(c: var CborParser; buf: var string): ?!void = + ## Read the text that the parser is positioned on into a string and advance. + try: + if c.kind != CborEventKind.cborText: + return failure(newCborError("Expected text, got " & $c.kind)) + buf.setLen c.intVal.int + if buf.len > 0: + let n = c.s.readData(buf[0].addr, buf.len) + if n != buf.len: + return failure(newCborError("truncated read of CBOR data")) + tryNext(c) + success() + except IOError as e: + return failure(e.msg) + except OSError as e: + return failure(e.msg) + + +proc nextText*(c: var CborParser): ?!string = + ## Read the text that the parser is positioned on into a string and advance. + var buf: string + let nextRes = nextText(c, buf) + if nextRes.isFailure: + return failure(nextRes.error) + + return success(buf) + +func arrayLen*(c: CborParser): int = + ## Return the length of the array that the parser is positioned on. + assert(c.kind == CborEventKind.cborArray, $c.kind) + c.intVal.int + +func mapLen*(c: CborParser): int = + ## Return the length of the map that the parser is positioned on. + assert(c.kind == CborEventKind.cborMap, $c.kind) + c.intVal.int + +func tag*(c: CborParser): uint64 = + ## Return the tag value the parser is positioned on. + assert(c.kind == CborEventKind.cborTag, $c.kind) + c.intVal + +proc skipNode*(c: var CborParser): ?!void = + ## Skip the item the parser is positioned on. + try: + case c.kind + of CborEventKind.cborEof: + return failure(newCborError("end of CBOR stream")) + of CborEventKind.cborPositive, CborEventKind.cborNegative, + CborEventKind.cborSimple: + return c.next() + of CborEventKind.cborBytes, CborEventKind.cborText: + if c.isIndefinite: + tryNext(c) + while c.kind != CborEventKind.cborBreak: + if c.kind != CborEventKind.cborBytes: + return failure(newCborError("expected bytes, got " & $c.kind)) + for _ in 1..c.intVal.int: discard readChar(c.s) + return c.next() + else: + for _ in 1..c.intVal.int: discard readChar(c.s) + return c.next() + of CborEventKind.cborArray: + if c.isIndefinite: + tryNext(c) + while c.kind != CborEventKind.cborBreak: + trySkip(c) + return c.next() + else: + let len = c.intVal + tryNext(c) + for i in 1..len: + trySkip(c) + of CborEventKind.cborMap: + let mapLen = c.intVal.int + if c.isIndefinite: + tryNext(c) + while c.kind != CborEventKind.cborBreak: + trySkip(c) + return c.next() + else: + tryNext(c) + for _ in 1 .. mapLen: + trySkip(c) + of CborEventKind.cborTag: + tryNext(c) + return c.skipNode() + of CborEventKind.cborFloat: + without f =? c.nextFloat(), error: + return failure(error) + of CborEventKind.cborBreak: + discard + success() + except OSError as e: + return failure(e.msg) + except IOError as e: + return failure(e.msg) + + + +proc nextNode*(c: var CborParser): ?!CborNode = + ## Parse the item the parser is positioned on into a ``CborNode``. + ## This is cheap for numbers or simple values but expensive + ## for nested types. + try: + var next: CborNode + case c.kind + of CborEventKind.cborEof: + return failure(newCborError("end of CBOR stream")) + of CborEventKind.cborPositive: + next = CborNode(kind: cborUnsigned, uint: c.intVal) + tryNext(c) + of CborEventKind.cborNegative: + next = CborNode(kind: cborNegative, int: -1 - c.intVal.int64) + tryNext(c) + of CborEventKind.cborBytes: + if c.isIndefinite: + next = CborNode(kind: cborBytes, bytes: newSeq[byte]()) + tryNext(c) + while c.kind != CborEventKind.cborBreak: + if c.kind != CborEventKind.cborBytes: + return failure(newCborError("Expected bytes, got " & $c.kind)) + let + chunkLen = c.intVal.int + pos = next.bytes.len + next.bytes.setLen(pos+chunkLen) + let n = c.s.readData(next.bytes[pos].addr, chunkLen) + if n != chunkLen: + return failure(newCborError("truncated read of CBOR data")) + tryNext(c) + else: + without rawBytes =? c.nextBytes(), error: + return failure(error) + next = CborNode(kind: cborBytes, bytes: rawBytes) + of CborEventKind.cborText: + if c.isIndefinite: + next = CborNode(kind: cborText, text: "") + tryNext(c) + while c.kind != CborEventKind.cborBreak: + if c.kind != CborEventKind.cborText: + return failure(newCborError("Expected text, got " & $c.kind)) + let + chunkLen = c.intVal.int + pos = next.text.len + next.text.setLen(pos+chunkLen) + let n = c.s.readData(next.text[pos].addr, chunkLen) + if n != chunkLen: + return failure(newCborError("truncated read of CBOR data")) + tryNext(c) + tryNext(c) + else: + without text =? c.nextText(), error: + return failure(error) + next = CborNode(kind: cborText, text: text) + of CborEventKind.cborArray: + next = CborNode(kind: cborArray, seq: newSeq[CborNode](c.intVal)) + if c.isIndefinite: + tryNext(c) + while c.kind != CborEventKind.cborBreak: + without node =? c.nextNode(), error: + return failure(error) + next.seq.add(node) + tryNext(c) + else: + tryNext(c) + for i in 0..next.seq.high: + without node =? c.nextNode(), error: + return failure(error) + next.seq[i] = node + of CborEventKind.cborMap: + let mapLen = c.intVal.int + next = CborNode(kind: cborMap, map: initOrderedTable[CborNode, CborNode]( + mapLen.nextPowerOfTwo)) + if c.isIndefinite: + tryNext(c) + while c.kind != CborEventKind.cborBreak: + without key =? c.nextNode(), error: + return failure(error) + without val =? c.nextNode(), error: + return failure(error) + next.map[key] = val + tryNext(c) + else: + tryNext(c) + for _ in 1 .. mapLen: + without key =? c.nextNode(), error: + return failure(error) + without val =? c.nextNode(), error: + return failure(error) + next.map[key] = val + of CborEventKind.cborTag: + let tag = c.intVal + tryNext(c) + without node =? c.nextNode(), error: + return failure(error) + next = node + next.tag = some tag + of CborEventKind.cborSimple: + case c.minor + of 24: + next = CborNode(kind: cborSimple, simple: c.intVal.uint8) + else: + next = CborNode(kind: cborSimple, simple: c.minor) + tryNext(c) + of CborEventKind.cborFloat: + without f =? c.nextFloat(), error: + return failure(error) + next = CborNode(kind: cborFloat, float: f) + of CborEventKind.cborBreak: + discard + success(next) + except OSError as e: + return failure(e.msg) + except IOError as e: + return failure(e.msg) + except Exception as e: + return failure(e.msg) + + +proc readCbor*(s: Stream): ?!CborNode = + ## Parse a stream into a CBOR object. + var parser: CborParser + parser.open(s) + tryNext(parser) + parser.nextNode() + +proc parseCbor*(s: string): ?!CborNode = + ## Parse a string into a CBOR object. + ## A wrapper over stream parsing. + readCbor(newStringStream s) + +proc `$`*(n: CborNode): string = + ## Get a ``CborNode`` in diagnostic notation. + result = "" + if n.tag.isSome: + result.add($n.tag.get) + result.add("(") + case n.kind + of cborUnsigned: + result.add $n.uint + of cborNegative: + result.add $n.int + of cborBytes: + result.add "h'" + for c in n.bytes: + result.add(c.toHex) + result.add "'" + of cborText: + result.add escape n.text + of cborArray: + result.add "[" + for i in 0.. 0: + result.add $(n.seq[n.seq.high]) + result.add "]" + of cborMap: + result.add "{" + let final = n.map.len + var i = 1 + for k, v in n.map.pairs: + result.add $k + result.add ": " + result.add $v + if i != final: + result.add ", " + inc i + result.add "}" + of cborTag: + discard + of cborSimple: + case n.simple + of 20: result.add "false" + of 21: result.add "true" + of 22: result.add "null" + of 23: result.add "undefined" + of 31: discard # break code for indefinite-length items + else: result.add "simple(" & $n.simple & ")" + of cborFloat: + case n.float.classify + of fcNan: + result.add "NaN" + of fcInf: + result.add "Infinity" + of fcNegInf: + result.add "-Infinity" + else: + result.add $n.float + of cborRaw: + without val =? parseCbor(n.raw), error: + return error.msg + result.add $val + if n.tag.isSome: + result.add(")") + + +proc getInt*(n: CborNode; default: int = 0): int = + ## Get the numerical value of a ``CborNode`` or a fallback. + case n.kind + of cborUnsigned: n.uint.int + of cborNegative: n.int.int + else: default + +proc parseDateText(n: CborNode): DateTime {.raises: [TimeParseError].} = + parse(n.text, timeFormat) + +proc parseTime(n: CborNode): Time = + case n.kind + of cborUnsigned, cborNegative: + result = fromUnix n.getInt + of cborFloat: + result = fromUnixFloat n.float + else: + assert false + +proc fromCborHook*(v: var DateTime; n: CborNode): bool = + ## Parse a `DateTime` from the tagged string representation + ## defined in RCF7049 section 2.4.1. + if n.tag.isSome: + try: + if n.tag.get == 0 and n.kind == cborText: + v = parseDateText(n) + result = true + elif n.tag.get == 1 and n.kind in {cborUnsigned, cborNegative, cborFloat}: + v = parseTime(n).utc + result = true + except ValueError: discard + +proc fromCborHook*(v: var Time; n: CborNode): bool = + ## Parse a `Time` from the tagged string representation + ## defined in RCF7049 section 2.4.1. + if n.tag.isSome: + try: + if n.tag.get == 0 and n.kind == cborText: + v = parseDateText(n).toTime + result = true + elif n.tag.get == 1 and n.kind in {cborUnsigned, cborNegative, cborFloat}: + v = parseTime(n) + result = true + except ValueError: discard + +func isTagged*(n: CborNode): bool = + ## Check if a CBOR item has a tag. + n.tag.isSome + +func hasTag*(n: CborNode; tag: Natural): bool = + ## Check if a CBOR item has a tag. + n.tag.isSome and n.tag.get == (uint64)tag + +proc `tag=`*(result: var CborNode; tag: Natural) = + ## Tag a CBOR item. + result.tag = some(tag.uint64) + +func tag*(n: CborNode): uint64 = + ## Get a CBOR item tag. + n.tag.get + +func isBool*(n: CborNode): bool = + (n.kind == cborSimple) and (n.simple in {20, 21}) + +func getBool*(n: CborNode; default = false): bool = + ## Get the boolean value of a ``CborNode`` or a fallback. + if n.kind == cborSimple: + case n.simple + of 20: false + of 21: true + else: default + else: + default + +func isNull*(n: CborNode): bool = + ## Return true if ``n`` is a CBOR null. + (n.kind == cborSimple) and (n.simple == 22) + +proc getUnsigned*(n: CborNode; default: uint64 = 0): uint64 = + ## Get the numerical value of a ``CborNode`` or a fallback. + case n.kind + of cborUnsigned: n.uint + of cborNegative: n.int.uint64 + else: default + +proc getSigned*(n: CborNode; default: int64 = 0): int64 = + ## Get the numerical value of a ``CborNode`` or a fallback. + case n.kind + of cborUnsigned: n.uint.int64 + of cborNegative: n.int + else: default + + +func getFloat*(n: CborNode; default = 0.0): float = + ## Get the floating-poing value of a ``CborNode`` or a fallback. + if n.kind == cborFloat: + n.float + else: + default + + +proc fromCbor*[T](v: var T; n: CborNode): bool = + ## Return `true` if `v` can be converted from a given `CborNode`. + ## Can be extended and overriden with `fromCborHook(v: var T; n: CborNode)` + ## for specific types of `T`. + when T is CborNode: + v = n + result = true + elif compiles(fromCborHook(v, n)): + result = fromCborHook(v, n) + elif T is distinct: + result = fromCbor(distinctBase v, n) + elif T is SomeUnsignedInt: + if n.kind == cborUnsigned: + v = T n.uint + result = v.BiggestUInt == n.uint + elif T is SomeSignedInt: + if n.kind == cborUnsigned: + v = T n.uint + result = v.BiggestUInt == n.uint + elif n.kind == cborNegative: + v = T n.int + result = v.BiggestInt == n.int + elif T is bool: + if n.isBool: + v = n.getBool + result = true + elif T is SomeFloat: + if n.kind == cborFloat: + v = T n.float + result = true + elif T is seq[byte]: + if n.kind == cborBytes: + v = n.bytes + result = true + elif T is string: + if n.kind == cborText: + v = n.text + result = true + elif T is seq: + if n.kind == cborArray: + result = true + v.setLen n.seq.len + for i, e in n.seq: + result = result and fromCbor(v[i], e) + if not result: + v.setLen 0 + break + elif T is tuple: + if n.kind == cborArray and n.seq.len == T.tupleLen: + result = true + var i: int + for f in fields(v): + result = result and fromCbor(f, n.seq[i]) + if not result: break + inc i + elif T is ref: + if n.isNull: + v = nil + result = true + else: + if isNil(v): new(v) + result = fromCbor(v[], n) + elif T is object: + if n.kind == cborMap: + result = true + var + i: int + key = CborNode(kind: cborText) + for s, _ in fieldPairs(v): + key.text = s + if not n.map.hasKey key: + result = false + break + else: + result = fromCbor(v.dot(s), n.map[key]) + if not result: break + inc i + result = result and (i == n.map.len) + +proc fromCborQ2*[T](v: var T; n: CborNode): ?!void = + ## Return a Result containing the value if `v` can be converted from a given `CborNode`, + ## or an error if conversion fails. + ## Can be extended and overriden with `fromCborHook(v: var T; n: CborNode)` + ## for specific types of `T`. + try: + when T is CborNode: + v = n + result = success() + elif compiles(fromCborHook(v, n)): + return fromCborHook(v, n) + elif T is distinct: + return fromCborQ2(distinctBase v, n) + elif T is SomeUnsignedInt: + exceptCborKind(T, {cborUnsigned}, n) + v = T n.uint + if v.BiggestUInt == n.uint: + return success() + else: + return failure(newCborError("Value overflow for unsigned integer")) + elif T is SomeSignedInt: + exceptCborKind(T, {cborUnsigned, cborNegative}, n) + if n.kind == cborUnsigned: + v = T n.uint + if v.BiggestUInt == n.uint: + return success() + else: + return failure(newCborError("Value overflow for un signed integer")) + elif n.kind == cborNegative: + v = T n.int + if v.BiggestInt == n.int: + return success() + else: + return failure(newCborError("Value overflow for signed integer")) + elif T is bool: + if not n.isBool: + return failure(newCborError("Expected boolean, got " & $n.kind)) + v = n.getBool + return success() + elif T is SomeFloat: + exceptCborKind(T, {cborFloat}, n) + v = T n.float + return success() + elif T is seq[byte]: + exceptCborKind(T, {cborBytes}, n) + v = n.bytes + return success() + elif T is string: + exceptCborKind(T, {cborText}, n) + v = n.text + return success() + elif T is seq: + exceptCborKind(T, {cborArray}, n) + v.setLen n.seq.len + for i, e in n.seq: + let itemResult = fromCborQ2(v[i], e) + if itemResult.isFailure: + v.setLen 0 + return failure(itemResult.error) + return success() + elif T is tuple: + exceptCborKind(T, {cborArray}, n) + if n.seq.len != T.tupleLen: + return failure(newCborError("Expected tuple of length " & $T.tupleLen)) + var i: int + for f in fields(v): + let itemResult = fromCborQ2(f, n.seq[i]) + if itemResult.isFailure: + return failure(itemResult.error) + inc i + return success() + elif T is ref: + if n.isNull: + v = nil + return success() + else: + if isNil(v): new(v) + return fromCborQ2(v[], n) + elif T is object: + exceptCborKind(T, {cborMap}, n) + var + i: int + key = CborNode(kind: cborText) + for s, _ in fieldPairs(v): + key.text = s + if not n.map.hasKey key: + return failure(newCborError("Missing field: " & s)) + else: + let fieldResult = fromCborQ2(v.dot(s), n.map[key]) + if fieldResult.isFailure: + return failure(fieldResult.error) + inc i + if i == n.map.len: + return success() + else: + return failure(newCborError("Extra fields in map")) + else: + return failure(newCborError("Unsupported type: " & $T)) + except Exception as e: + return failure(newCborError(e.msg)) diff --git a/serde/cbor/errors.nim b/serde/cbor/errors.nim new file mode 100644 index 0000000..f375a1a --- /dev/null +++ b/serde/cbor/errors.nim @@ -0,0 +1,37 @@ +import ../utils/types +import ./types +import std/sets + + +proc newUnexpectedKindError*( + expectedType: type, expectedKinds: string, cbor: CborNode +): ref UnexpectedKindError = + newException( + UnexpectedKindError, + "deserialization to " & $expectedType & " failed: expected " & + expectedKinds & + " but got " & $cbor.kind, + ) + +proc newUnexpectedKindError*( + expectedType: type, expectedKinds: set[CborEventKind], cbor: CborNode +): ref UnexpectedKindError = + newUnexpectedKindError(expectedType, $expectedKinds, cbor) + +proc newUnexpectedKindError*( + expectedType: type, expectedKind: CborEventKind, cbor: CborNode +): ref UnexpectedKindError = + newUnexpectedKindError(expectedType, {expectedKind}, cbor) + +proc newUnexpectedKindError*( + expectedType: type, expectedKinds: set[CborNodeKind], cbor: CborNode +): ref UnexpectedKindError = + newUnexpectedKindError(expectedType, $expectedKinds, cbor) + +proc newUnexpectedKindError*( + expectedType: type, expectedKind: CborNodeKind, cbor: CborNode +): ref UnexpectedKindError = + newUnexpectedKindError(expectedType, {expectedKind}, cbor) + +proc newCborError*(msg: string): ref CborParseError = + newException(CborParseError, msg) diff --git a/serde/cbor/helpers.nim b/serde/cbor/helpers.nim new file mode 100644 index 0000000..8450534 --- /dev/null +++ b/serde/cbor/helpers.nim @@ -0,0 +1,42 @@ + +import ./types +import ./errors +from macros import newDotExpr, newIdentNode, strVal + +template exceptCborKind*(expectedType: type, expectedKinds: set[CborNodeKind], + cbor: CborNode) = + if cbor.kind notin expectedKinds: + return failure(newUnexpectedKindError(expectedType, expectedKinds, cbor)) + +template exceptCborKind*(expectedType: type, expectedKind: CborNodeKind, + cbor: CborNode) = + exceptCborKind(expectedType, {expectedKind}, cbor) + +template exceptCborKind*(expectedType: type, expectedKinds: set[CborEventKind], + cbor: CborNode) = + if cbor.kind notin expectedKinds: + return failure(newUnexpectedKindError(expectedType, expectedKinds, cbor)) + +template exceptCborKind*(expectedType: type, expectedKind: CborEventKind, + cbor: CborNode) = + exceptCborKind(expectedType, {expectedKind}, cbor) + +macro dot*(obj: object, fld: string): untyped = + ## Turn ``obj.dot("fld")`` into ``obj.fld``. + newDotExpr(obj, newIdentNode(fld.strVal)) + + +func floatSingle*(half: uint16): float32 = + ## Convert a 16-bit float to 32-bits. + func ldexp(x: float64; exponent: int): float64 {.importc: "ldexp", + header: "".} + let + exp = (half shr 10) and 0x1f + mant = float64(half and 0x3ff) + val = if exp == 0: + ldexp(mant, -24) + elif exp != 31: + ldexp(mant + 1024, exp.int - 25) + else: + if mant == 0: Inf else: NaN + if (half and 0x8000) == 0: val else: -val diff --git a/serde/cbor/jsonhook.nim b/serde/cbor/jsonhook.nim new file mode 100644 index 0000000..21e7a7a --- /dev/null +++ b/serde/cbor/jsonhook.nim @@ -0,0 +1,42 @@ +import std/[base64, tables] +import ../json/stdjson +import ./types +import ./errors +import ./deserializer + +proc toJsonHook*(n: CborNode): JsonNode = + case n.kind: + of cborUnsigned: + newJInt n.uint.BiggestInt + of cborNegative: + newJInt n.int.BiggestInt + of cborBytes: + newJString base64.encode(cast[string](n.bytes), safe = true) + of cborText: + newJString n.text + of cborArray: + let a = newJArray() + for e in n.seq.items: + a.add(e.toJsonHook) + a + of cborMap: + let o = newJObject() + for k, v in n.map.pairs: + if k.kind == cborText: + o[k.text] = v.toJsonHook + else: + o[$k] = v.toJsonHook + o + of cborTag: nil + of cborSimple: + if n.isBool: + newJBool(n.getBool()) + elif n.isNull: + newJNull() + else: nil + of cborFloat: + newJFloat n.float + of cborRaw: + without parsed =? parseCbor(n.raw), error: + raise newCborError(error.msg) + toJsonHook(parsed) diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index 9b39e60..4422958 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -1,12 +1,42 @@ {.push checks: off.} -import ../utils/cbor +import std/[streams, options, tables, typetraits, math, endians, times, base64] +import ./types + +func isHalfPrecise(single: float32): bool = + # TODO: check for subnormal false-positives + let val = cast[uint32](single) + if val == 0 or val == (1'u32 shl 31): + result = true + else: + let + exp = int32((val and (0xff'u32 shl 23)) shr 23) - 127 + mant = val and 0x7fffff'u32 + if -25 < exp and exp < 16 and (mant and 0x1fff) == 0: + result = true + +func floatHalf(single: float32): uint16 = + ## Convert a 32-bit float to 16-bits. + let + val = cast[uint32](single) + exp = val and 0x7f800000 + mant = val and 0x7fffff + sign = uint16(val shr 16) and (1 shl 15) + let + unbiasedExp = int32(exp shr 23) - 127 + halfExp = unbiasedExp + 15 + if halfExp < 1: + if 14 - halfExp < 25: + result = sign or uint16((mant or 0x800000) shr uint16(14 - halfExp)) + else: + result = sign or uint16(halfExp shl 10) or uint16(mant shr 13) func initialByte(major, minor: Natural): uint8 {.inline.} = uint8((major shl 5) or (minor and 0b11111)) proc writeInitial[T: SomeInteger](str: Stream; m: uint8; n: T) = ## Write the initial integer of a CBOR item. + let m = m shl 5 when T is byte: if n < 24: @@ -36,7 +66,7 @@ proc writeInitial[T: SomeInteger](str: Stream; m: uint8; n: T) = {.unroll.} str.write((uint8)n shr i) str.write((uint8)n) -{.pop.} +# {.pop.} proc writeCborArrayLen*(str: Stream; len: Natural) = ## Write a marker to the stream that initiates an array of ``len`` items. @@ -71,6 +101,8 @@ proc writeCbor*(str: Stream; buf: pointer; len: int) = str.writeInitial(BytesMajor, len) if len > 0: str.writeData(buf, len) +proc isSorted*(n: CborNode): bool {.gcsafe.} + proc writeCbor*[T](str: Stream; v: T) = ## Write the CBOR binary representation of a `T` to a `Stream`. ## The behavior of this procedure can be extended or overriden @@ -219,3 +251,119 @@ proc encode*[T](v: T): string = let s = newStringStream() s.writeCbor(v) s.data + +proc toRaw*(n: CborNode): CborNode = + ## Reduce a CborNode to a string of bytes. + if n.kind == cborRaw: n + else: CborNode(kind: cborRaw, raw: encode(n)) + +proc isSorted(n: CborNode): bool = + ## Check if the item is sorted correctly. + var lastRaw = "" + for key in n.map.keys: + let thisRaw = key.toRaw.raw + if lastRaw != "": + if cmp(lastRaw, thisRaw) > 0: return false + lastRaw = thisRaw + true + +proc sort*(n: var CborNode) = + ## Sort a CBOR map object. + var tmp = initOrderedTable[CborNode, CborNode](n.map.len.nextPowerOfTwo) + for key, val in n.map.mpairs: + tmp[key.toRaw] = move(val) + sort(tmp) do (x, y: tuple[k: CborNode; v: CborNode]) -> int: + result = cmp(x.k.raw, y.k.raw) + n.map = move tmp + +proc writeCborHook*(str: Stream; dt: DateTime) = + ## Write a `DateTime` using the tagged string representation + ## defined in RCF7049 section 2.4.1. + writeCborTag(str, 0) + writeCbor(str, format(dt, timeFormat)) + +proc writeCborHook*(str: Stream; t: Time) = + ## Write a `Time` using the tagged numerical representation + ## defined in RCF7049 section 2.4.1. + writeCborTag(str, 1) + writeCbor(str, t.toUnix) + +func toCbor*(x: CborNode): CborNode = x + +func toCbor*(x: SomeInteger): CborNode = + if x > 0: + CborNode(kind: cborUnsigned, uint: x.uint64) + else: + CborNode(kind: cborNegative, int: x.int64) + +func toCbor*(x: openArray[byte]): CborNode = + CborNode(kind: cborBytes, bytes: @x) + +func toCbor*(x: string): CborNode = + CborNode(kind: cborText, text: x) + +func toCbor*(x: openArray[CborNode]): CborNode = + CborNode(kind: cborArray, seq: @x) + +func toCbor*(pairs: openArray[(CborNode, CborNode)]): CborNode = + CborNode(kind: cborMap, map: pairs.toOrderedTable) + +func toCbor*(tag: uint64; val: CborNode): CborNode = + result = toCbor(val) + result.tag = some(tag) + +func toCbor*(x: bool): CborNode = + case x + of false: + CborNode(kind: cborSimple, simple: 20) + of true: + CborNode(kind: cborSimple, simple: 21) + +func toCbor*(x: SomeFloat): CborNode = + CborNode(kind: cborFloat, float: x.float64) + +func toCbor*(x: pointer): CborNode = + ## A hack to produce a CBOR null item. + assert(x.isNil) + CborNode(kind: cborSimple, simple: 22) + +func initCborBytes*[T: char|byte](buf: openArray[T]): CborNode = + ## Create a CBOR byte string from `buf`. + result = CborNode(kind: cborBytes, bytes: newSeq[byte](buf.len)) + for i in 0..= 0.10.13 & < 0.11.0" task test, "Run the test suite": exec "nimble install -d -y" exec "nim c -r test" + \ No newline at end of file From beac0e2a329dbd156827271a2d2d50bde9337b8d Mon Sep 17 00:00:00 2001 From: munna0908 Date: Wed, 21 May 2025 03:41:35 +0530 Subject: [PATCH 03/20] cleanup test files --- serde/cbor/deserializer.nim | 91 +------ tests/cbor/testDeserialize.nim | 260 -------------------- tests/cbor/testObjects.nim | 177 +++++++++++++ tests/cbor/{test.nim => testprimitives.nim} | 8 + tests/test.nim | 4 +- 5 files changed, 193 insertions(+), 347 deletions(-) delete mode 100644 tests/cbor/testDeserialize.nim create mode 100644 tests/cbor/testObjects.nim rename tests/cbor/{test.nim => testprimitives.nim} (89%) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index cdff43b..56a5300 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -580,86 +580,7 @@ func getFloat*(n: CborNode; default = 0.0): float = default -proc fromCbor*[T](v: var T; n: CborNode): bool = - ## Return `true` if `v` can be converted from a given `CborNode`. - ## Can be extended and overriden with `fromCborHook(v: var T; n: CborNode)` - ## for specific types of `T`. - when T is CborNode: - v = n - result = true - elif compiles(fromCborHook(v, n)): - result = fromCborHook(v, n) - elif T is distinct: - result = fromCbor(distinctBase v, n) - elif T is SomeUnsignedInt: - if n.kind == cborUnsigned: - v = T n.uint - result = v.BiggestUInt == n.uint - elif T is SomeSignedInt: - if n.kind == cborUnsigned: - v = T n.uint - result = v.BiggestUInt == n.uint - elif n.kind == cborNegative: - v = T n.int - result = v.BiggestInt == n.int - elif T is bool: - if n.isBool: - v = n.getBool - result = true - elif T is SomeFloat: - if n.kind == cborFloat: - v = T n.float - result = true - elif T is seq[byte]: - if n.kind == cborBytes: - v = n.bytes - result = true - elif T is string: - if n.kind == cborText: - v = n.text - result = true - elif T is seq: - if n.kind == cborArray: - result = true - v.setLen n.seq.len - for i, e in n.seq: - result = result and fromCbor(v[i], e) - if not result: - v.setLen 0 - break - elif T is tuple: - if n.kind == cborArray and n.seq.len == T.tupleLen: - result = true - var i: int - for f in fields(v): - result = result and fromCbor(f, n.seq[i]) - if not result: break - inc i - elif T is ref: - if n.isNull: - v = nil - result = true - else: - if isNil(v): new(v) - result = fromCbor(v[], n) - elif T is object: - if n.kind == cborMap: - result = true - var - i: int - key = CborNode(kind: cborText) - for s, _ in fieldPairs(v): - key.text = s - if not n.map.hasKey key: - result = false - break - else: - result = fromCbor(v.dot(s), n.map[key]) - if not result: break - inc i - result = result and (i == n.map.len) - -proc fromCborQ2*[T](v: var T; n: CborNode): ?!void = +proc fromCbor*[T](v: var T; n: CborNode): ?!void = ## Return a Result containing the value if `v` can be converted from a given `CborNode`, ## or an error if conversion fails. ## Can be extended and overriden with `fromCborHook(v: var T; n: CborNode)` @@ -671,7 +592,7 @@ proc fromCborQ2*[T](v: var T; n: CborNode): ?!void = elif compiles(fromCborHook(v, n)): return fromCborHook(v, n) elif T is distinct: - return fromCborQ2(distinctBase v, n) + return fromCbor(distinctBase v, n) elif T is SomeUnsignedInt: exceptCborKind(T, {cborUnsigned}, n) v = T n.uint @@ -714,7 +635,7 @@ proc fromCborQ2*[T](v: var T; n: CborNode): ?!void = exceptCborKind(T, {cborArray}, n) v.setLen n.seq.len for i, e in n.seq: - let itemResult = fromCborQ2(v[i], e) + let itemResult = fromCbor(v[i], e) if itemResult.isFailure: v.setLen 0 return failure(itemResult.error) @@ -725,7 +646,7 @@ proc fromCborQ2*[T](v: var T; n: CborNode): ?!void = return failure(newCborError("Expected tuple of length " & $T.tupleLen)) var i: int for f in fields(v): - let itemResult = fromCborQ2(f, n.seq[i]) + let itemResult = fromCbor(f, n.seq[i]) if itemResult.isFailure: return failure(itemResult.error) inc i @@ -736,7 +657,7 @@ proc fromCborQ2*[T](v: var T; n: CborNode): ?!void = return success() else: if isNil(v): new(v) - return fromCborQ2(v[], n) + return fromCbor(v[], n) elif T is object: exceptCborKind(T, {cborMap}, n) var @@ -747,7 +668,7 @@ proc fromCborQ2*[T](v: var T; n: CborNode): ?!void = if not n.map.hasKey key: return failure(newCborError("Missing field: " & s)) else: - let fieldResult = fromCborQ2(v.dot(s), n.map[key]) + let fieldResult = fromCbor(v.dot(s), n.map[key]) if fieldResult.isFailure: return failure(fieldResult.error) inc i diff --git a/tests/cbor/testDeserialize.nim b/tests/cbor/testDeserialize.nim deleted file mode 100644 index 320f617..0000000 --- a/tests/cbor/testDeserialize.nim +++ /dev/null @@ -1,260 +0,0 @@ -# File: /Users/rahul/Work/repos/nim-serde/tests/cbor_questionable.nim - -import std/unittest -import std/options -import std/streams -import pkg/serde -import pkg/questionable -import pkg/questionable/results - -# Custom type for testing -type - CustomPoint = object - x: int - y: int - - CustomColor = enum - Red, Green, Blue - - CustomObject = object - name: string - point: CustomPoint - color: CustomColor - - Person = object - name: string - age: int - isActive: bool - - Inner = object - s: string - nums: seq[int] - - CompositeNested = object - u: uint64 - n: int - b: seq[byte] - t: string - arr: seq[int] - tag: float - flag: bool - inner: Inner - innerArr: seq[Inner] - coordinates: tuple[x: int, y: int, label: string] - refInner: ref Inner - -proc fromCborHook*(v: var CustomColor, n: CborNode): ?!void = - if n.kind == cborNegative: - v = CustomColor(n.int) - result = success() - else: - result = failure(newSerdeError("Expected signed integer, got " & $n.kind)) - -# Custom fromCborHook for CustomPoint -proc fromCborHook*(v: var CustomPoint, n: CborNode): ?!void = - if n.kind == cborArray and n.seq.len == 2: - var x, y: int - let xResult = fromCborQ2(x, n.seq[0]) - if xResult.isFailure: - return failure(xResult.error) - - let yResult = fromCborQ2(y, n.seq[1]) - if yResult.isFailure: - return failure(yResult.error) - - v = CustomPoint(x: x, y: y) - result = success() - else: - result = failure(newSerdeError("Expected array of length 2 for CustomPoint")) - -# Helper function to create CBOR data for testing -proc createPointCbor(x, y: int): CborNode = - result = CborNode(kind: cborArray) - result.seq = @[ - CborNode(kind: cborUnsigned, uint: x.uint64), - CborNode(kind: cborUnsigned, uint: y.uint64) - ] - -proc createObjectCbor(name: string, point: CustomPoint, - color: CustomColor): CborNode = - result = CborNode(kind: cborMap) - result.map = initOrderedTable[CborNode, CborNode]() - - # Add name field - result.map[CborNode(kind: cborText, text: "name")] = - CborNode(kind: cborText, text: name) - - # Add point field - result.map[CborNode(kind: cborText, text: "point")] = - createPointCbor(point.x, point.y) - - # Add color field - result.map[CborNode(kind: cborText, text: "color")] = - CborNode(kind: cborNegative, int: color.int) - -suite "CBOR deserialization with Questionable": - test "fromCborQ2 with primitive types": - # Test with integer - block: - var intValue: int - let node = CborNode(kind: cborUnsigned, uint: 42.uint64) - let result = fromCborQ2(intValue, node) - - check result.isSuccess - check intValue == 42 - - # Test with string - block: - var strValue: string - let node = CborNode(kind: cborText, text: "hello") - let result = fromCborQ2(strValue, node) - - check result.isSuccess - check strValue == "hello" - - # Test with error case - block: - var intValue: int - let node = CborNode(kind: cborText, text: "not an int") - let result = fromCborQ2(intValue, node) - - check result.isFailure - check $result.error.msg == "deserialization to int failed: expected {cborUnsigned, cborNegative} but got cborText" - - test "parseCborAs with valid input": - # Create a valid CBOR object for a Person - var mapNode = CborNode(kind: cborMap) - mapNode.map = initOrderedTable[CborNode, CborNode]() - mapNode.map[CborNode(kind: cborText, text: "a")] = CborNode( - kind: cborText, text: "John Doe") - mapNode.map[CborNode(kind: cborText, text: "b")] = CborNode( - kind: cborUnsigned, uint: 30) - mapNode.map[CborNode(kind: cborText, text: "c")] = CborNode( - kind: cborSimple, simple: 21) # true - var p1: Person - p1.name = "John Doe" - p1.age = 30 - p1.isActive = true - - let stream = newStringStream() - stream.writeCbor(p1) - let cborData = stream.data - - # var cborNode = parseCbor(cborData) - # check cborNode.isSuccess - # echo cborNode.tryError.msg - - without parsedNode =? parseCbor(cborData), error: - echo error.msg - - # Parse directly to Person object - var person: Person - let result = fromCborQ2(person, parsedNode) - - check result.isSuccess - check person.name == "John Doe" - check person.age == 30 - check person.isActive == true - - test "fromCborQ2 with custom hook": - # Test with valid point data - block: - var point: CustomPoint - let node = createPointCbor(10, 20) - let result = fromCborQ2(point, node) - - check result.isSuccess - check point.x == 10 - check point.y == 20 - - # Test with invalid point data - block: - var point: CustomPoint - let elements = @[toCbor(10)] - let node = toCbor(elements) - let result = fromCborQ2(point, node) - - check result.isFailure - # check "Expected array of length 2" in $result.error.msg - - test "fromCborQ2 with complex object": - # Create a complex object - let point = CustomPoint(x: 15, y: 25) - # let obj = CustomObject(name: "Test Object", point: point, color: Green) - - # Create CBOR representation - let node = createObjectCbor("Test Object", point, Green) - - # Deserialize - var deserializedObj: CustomObject - # Check result - let result = fromCborQ2(deserializedObj, node) - check result.isSuccess - check deserializedObj.name == "Test Object" - check deserializedObj.point.x == 15 - check deserializedObj.point.y == 25 - check deserializedObj.color == Green - - suite "CBOR round-trip for nested composite object": - test "serialize and parse nested composite type": - var refObj = new Inner - refObj.s = "refInner" - refObj.nums = @[30, 40] - var original = CompositeNested( - u: 42, - n: -99, - b: @[byte 1, byte 2], - t: "hi", - arr: @[1, 2, 3], - tag: 1.5, - flag: true, - inner: Inner(s: "inner!", nums: @[10, 20]), - innerArr: @[ - Inner(s: "first", nums: @[1, 2]), - Inner(s: "second", nums: @[3, 4, 5]) - ], - coordinates: (x: 10, y: 20, label: "test"), - refInner: refObj - ) - - # Serialize to CBOR - let stream = newStringStream() - stream.writeCbor(original) - let cborData = stream.data - # Parse CBOR back to CborNode - let parseResult = parseCbor(cborData) - check parseResult.isSuccess - let node = parseResult.tryValue - - # Deserialize to CompositeNested object - var roundtrip: CompositeNested - let deserResult = fromCborQ2(roundtrip, node) - check deserResult.isSuccess - - # Check top-level fields - check roundtrip.u == original.u - check roundtrip.n == original.n - check roundtrip.b == original.b - check roundtrip.t == original.t - check roundtrip.arr == original.arr - check abs(roundtrip.tag - original.tag) < 1e-6 - check roundtrip.flag == original.flag - - # Check nested object - check roundtrip.inner.s == original.inner.s - check roundtrip.inner.nums == original.inner.nums - - # Check nested array of objects - check roundtrip.innerArr.len == original.innerArr.len - for i in 0.. Date: Thu, 22 May 2025 12:03:22 +0530 Subject: [PATCH 04/20] improve error handling for cbor serialization --- serde/cbor/deserializer.nim | 2 +- serde/cbor/serializer.nim | 487 +++++++++++++++++++----------------- tests/cbor/testObjects.nim | 17 +- 3 files changed, 272 insertions(+), 234 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 56a5300..3d406f9 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -9,11 +9,11 @@ import pkg/questionable/results export results export types +{.push raises: [].} func isIndefinite*(c: CborParser): bool {.inline.} = c.minor == 31 ## Return true if the parser is positioned on an item of indefinite length. -{.push raises: [].} proc open*(c: var CborParser; s: Stream) = ## Begin parsing a stream of CBOR in binary form. ## The parser will be initialized in an EOF state, call diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index 4422958..4165547 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -1,8 +1,11 @@ -{.push checks: off.} - import std/[streams, options, tables, typetraits, math, endians, times, base64] +import pkg/questionable +import pkg/questionable/results +import ../utils/errors import ./types +{.push raises: [].} + func isHalfPrecise(single: float32): bool = # TODO: check for subnormal false-positives let val = cast[uint32](single) @@ -34,298 +37,335 @@ func floatHalf(single: float32): uint16 = func initialByte(major, minor: Natural): uint8 {.inline.} = uint8((major shl 5) or (minor and 0b11111)) -proc writeInitial[T: SomeInteger](str: Stream; m: uint8; n: T) = +proc writeInitial[T: SomeInteger](str: Stream; m: uint8; n: T): ?!void = ## Write the initial integer of a CBOR item. - - let m = m shl 5 - when T is byte: - if n < 24: - str.write(m or n.uint8) + try: + let m = m shl 5 + when T is byte: + if n < 24: + str.write(m or n.uint8) + else: + str.write(m or 24'u8) + str.write(n) else: - str.write(m or 24'u8) - str.write(n) - else: - if n < 24: - str.write(m or n.uint8) - elif uint64(n) <= uint64(uint8.high): - str.write(m or 24'u8) - str.write(n.uint8) - elif uint64(n) <= uint64(uint16.high): - str.write(m or 25'u8) - str.write((uint8)n shr 8) - str.write((uint8)n) - elif uint64(n) <= uint64(uint32.high): - str.write(m or 26'u8) - for i in countdown(24, 8, 8): - {.unroll.} - str.write((uint8)n shr i) - str.write((uint8)n) - else: - str.write(m or 27'u8) - for i in countdown(56, 8, 8): - {.unroll.} - str.write((uint8)n shr i) - str.write((uint8)n) -# {.pop.} + if n < 24: + str.write(m or n.uint8) + elif uint64(n) <= uint64(uint8.high): + str.write(m or 24'u8) + str.write(n.uint8) + elif uint64(n) <= uint64(uint16.high): + str.write(m or 25'u8) + str.write((uint8)n shr 8) + str.write((uint8)n) + elif uint64(n) <= uint64(uint32.high): + str.write(m or 26'u8) + for i in countdown(24, 8, 8): + {.unroll.} + str.write((uint8)n shr i) + str.write((uint8)n) + else: + str.write(m or 27'u8) + for i in countdown(56, 8, 8): + {.unroll.} + str.write((uint8)n shr i) + str.write((uint8)n) + success() + except IOError as e: + return failure(e.msg) + except OSError as o: + return failure(o.msg) -proc writeCborArrayLen*(str: Stream; len: Natural) = +proc writeCborArrayLen*(str: Stream; len: Natural): ?!void = ## Write a marker to the stream that initiates an array of ``len`` items. str.writeInitial(4, len) -proc writeCborIndefiniteArrayLen*(str: Stream) = +proc writeCborIndefiniteArrayLen*(str: Stream): ?!void = ## Write a marker to the stream that initiates an array of indefinite length. ## Indefinite length arrays are composed of an indefinite amount of arrays ## of definite lengths. - str.write(initialByte(4, 31)) + catch str.write(initialByte(4, 31)) -proc writeCborMapLen*(str: Stream; len: Natural) = +proc writeCborMapLen*(str: Stream; len: Natural): ?!void = ## Write a marker to the stream that initiates an map of ``len`` pairs. str.writeInitial(5, len) -proc writeCborIndefiniteMapLen*(str: Stream) = +proc writeCborIndefiniteMapLen*(str: Stream): ?!void = ## Write a marker to the stream that initiates a map of indefinite length. ## Indefinite length maps are composed of an indefinite amount of maps ## of definite length. - str.write(initialByte(5, 31)) + catch str.write(initialByte(5, 31)) -proc writeCborBreak*(str: Stream) = +proc writeCborBreak*(str: Stream): ?!void = ## Write a marker to the stream that ends an indefinite array or map. - str.write(initialByte(7, 31)) + catch str.write(initialByte(7, 31)) -proc writeCborTag*(str: Stream; tag: Natural) {.inline.} = +proc writeCborTag*(str: Stream; tag: Natural): ?!void {.inline.} = ## Write a tag for the next CBOR item to a binary stream. str.writeInitial(6, tag) -proc writeCbor*(str: Stream; buf: pointer; len: int) = +proc writeCbor*(str: Stream; buf: pointer; len: int): ?!void = ## Write a raw buffer to a CBOR `Stream`. - str.writeInitial(BytesMajor, len) - if len > 0: str.writeData(buf, len) + ?str.writeInitial(BytesMajor, len) + if len > 0: + return catch str.writeData(buf, len) -proc isSorted*(n: CborNode): bool {.gcsafe.} +proc isSorted*(n: CborNode): ?!bool {.gcsafe.} -proc writeCbor*[T](str: Stream; v: T) = +proc writeCbor*[T](str: Stream; v: T): ?!void = ## Write the CBOR binary representation of a `T` to a `Stream`. ## The behavior of this procedure can be extended or overriden ## by defining `proc writeCborHook(str: Stream; v: T)` for specific ## types `T`. - when T is CborNode: - if v.tag.isSome: - str.writeCborTag(v.tag.get) - case v.kind: - of cborUnsigned: - str.writeCbor(v.uint) - of cborNegative: - str.writeCbor(v.int) - of cborBytes: - str.writeInitial(cborBytes.uint8, v.bytes.len) - for b in v.bytes.items: - str.write(b) - of cborText: - str.writeInitial(cborText.uint8, v.text.len) - str.write(v.text) - of cborArray: - str.writeInitial(4, v.seq.len) - for e in v.seq: - str.writeCbor(e) - of cborMap: - assert(v.isSorted, "refusing to write unsorted map to stream") - str.writeInitial(5, v.map.len) - for k, f in v.map.pairs: - str.writeCbor(k) - str.writeCbor(f) - of cborTag: - discard - of cborSimple: - if v.simple > 31'u or v.simple == 24: - str.write(initialByte(cborSimple.uint8, 24)) - str.write(v.simple) - else: - str.write(initialByte(cborSimple.uint8, v.simple)) - of cborFloat: - str.writeCbor(v.float) - of cborRaw: - str.write(v.raw) - elif compiles(writeCborHook(str, v)): - writeCborHook(str, v) - elif T is SomeUnsignedInt: - str.writeInitial(0, v) - elif T is SomeSignedInt: - if v < 0: - # Major type 1 - str.writeInitial(1, -1 - v) - else: - # Major type 0 - str.writeInitial(0, v) - elif T is seq[byte]: - str.writeInitial(BytesMajor, v.len) - if v.len > 0: - str.writeData(unsafeAddr v[0], v.len) - elif T is openArray[char | uint8 | int8]: - str.writeInitial(BytesMajor, v.len) - if v.len > 0: - str.writeData(unsafeAddr v[0], v.len) - elif T is string: - str.writeInitial(TextMajor, v.len) - str.write(v) - elif T is array | seq: - str.writeInitial(4, v.len) - for e in v.items: - writeCbor(str, e) - elif T is tuple: - str.writeInitial(4, T.tupleLen) - for f in v.fields: str.writeCbor(f) - elif T is ptr | ref: - if system.`==`(v, nil): - # Major type 7 - str.write(Null) - else: writeCbor(str, v[]) - elif T is object: - var n: uint - for _, _ in v.fieldPairs: - inc n - str.writeInitial(5, n) - for k, f in v.fieldPairs: - str.writeCbor(k) - str.writeCbor(f) - elif T is bool: - str.write(initialByte(7, (if v: 21 else: 20))) - elif T is SomeFloat: - case v.classify - of fcNormal, fcSubnormal: - let single = v.float32 - if single.float64 == v.float64: - if single.isHalfPrecise: - let half = floatHalf(single) - str.write(initialByte(7, 25)) - when system.cpuEndian == bigEndian: - str.write(half) - else: - var be: uint16 - swapEndian16 be.addr, half.unsafeAddr - str.write(be) + try: + when T is CborNode: + if v.tag.isSome: + return str.writeCborTag(v.tag.get) + case v.kind: + of cborUnsigned: + return str.writeCbor(v.uint) + of cborNegative: + return str.writeCbor(v.int) + of cborBytes: + ?str.writeInitial(cborBytes.uint8, v.bytes.len) + for b in v.bytes.items: + str.write(b) + of cborText: + ?str.writeInitial(cborText.uint8, v.text.len) + str.write(v.text) + of cborArray: + ?str.writeInitial(4, v.seq.len) + for e in v.seq: + ?str.writeCbor(e) + of cborMap: + without isSortedRes =? v.isSorted, error: + return failure(error) + if not isSortedRes: + return failure(newSerdeError("refusing to write unsorted map to stream")) + ?str.writeInitial(5, v.map.len) + for k, f in v.map.pairs: + ?str.writeCbor(k) + ?str.writeCbor(f) + of cborTag: + discard + of cborSimple: + if v.simple > 31'u or v.simple == 24: + str.write(initialByte(cborSimple.uint8, 24)) + str.write(v.simple) else: - str.write initialByte(7, 26) - when system.cpuEndian == bigEndian: - str.write(single) - else: - var be: uint32 - swapEndian32 be.addr, single.unsafeAddr - str.write(be) + str.write(initialByte(cborSimple.uint8, v.simple)) + of cborFloat: + return str.writeCbor(v.float) + of cborRaw: + str.write(v.raw) + elif compiles(writeCborHook(str, v)): + writeCborHook(str, v) + elif T is SomeUnsignedInt: + ?str.writeInitial(0, v) + elif T is SomeSignedInt: + if v < 0: + # Major type 1 + ?str.writeInitial(1, -1 - v) else: - str.write initialByte(7, 27) - when system.cpuEndian == bigEndian: - str.write(v) + # Major type 0 + ?str.writeInitial(0, v) + elif T is seq[byte]: + ?str.writeInitial(BytesMajor, v.len) + if v.len > 0: + str.writeData(unsafeAddr v[0], v.len) + elif T is openArray[char | uint8 | int8]: + ?str.writeInitial(BytesMajor, v.len) + if v.len > 0: + str.writeData(unsafeAddr v[0], v.len) + elif T is string: + ?str.writeInitial(TextMajor, v.len) + str.write(v) + elif T is array | seq: + ?str.writeInitial(4, v.len) + for e in v.items: + ?str.writeCbor(e) + elif T is tuple: + ?str.writeInitial(4, T.tupleLen) + for f in v.fields: ?str.writeCbor(f) + elif T is ptr | ref: + if system.`==`(v, nil): + # Major type 7 + str.write(Null) + else: ?str.writeCbor(v[]) + elif T is object: + var n: uint + for _, _ in v.fieldPairs: + inc n + ?str.writeInitial(5, n) + for k, f in v.fieldPairs: + ?str.writeCbor(k) + ?str.writeCbor(f) + elif T is bool: + str.write(initialByte(7, (if v: 21 else: 20))) + elif T is SomeFloat: + case v.classify + of fcNormal, fcSubnormal: + let single = v.float32 + if single.float64 == v.float64: + if single.isHalfPrecise: + let half = floatHalf(single) + str.write(initialByte(7, 25)) + when system.cpuEndian == bigEndian: + str.write(half) + else: + var be: uint16 + swapEndian16 be.addr, half.unsafeAddr + str.write(be) + else: + str.write initialByte(7, 26) + when system.cpuEndian == bigEndian: + str.write(single) + else: + var be: uint32 + swapEndian32 be.addr, single.unsafeAddr + str.write(be) else: - var be: float64 - swapEndian64 be.addr, v.unsafeAddr - str.write be - return - of fcZero: - str.write initialByte(7, 25) - str.write((char)0x00) - of fcNegZero: - str.write initialByte(7, 25) - str.write((char)0x80) - of fcInf: - str.write initialByte(7, 25) - str.write((char)0x7c) - of fcNan: - str.write initialByte(7, 25) - str.write((char)0x7e) - of fcNegInf: - str.write initialByte(7, 25) - str.write((char)0xfc) - str.write((char)0x00) + str.write initialByte(7, 27) + when system.cpuEndian == bigEndian: + str.write(v) + else: + var be: float64 + swapEndian64 be.addr, v.unsafeAddr + str.write be -proc writeCborArray*(str: Stream; args: varargs[CborNode, toCbor]) = + return success() + of fcZero: + str.write initialByte(7, 25) + str.write((char)0x00) + of fcNegZero: + str.write initialByte(7, 25) + str.write((char)0x80) + of fcInf: + str.write initialByte(7, 25) + str.write((char)0x7c) + of fcNan: + str.write initialByte(7, 25) + str.write((char)0x7e) + of fcNegInf: + str.write initialByte(7, 25) + str.write((char)0xfc) + str.write((char)0x00) + success() + except IOError as io: + return failure(io.msg) + except OSError as os: + return failure(os.msg) + +proc writeCborArray*(str: Stream; args: varargs[CborNode, toCbor]): ?!void = ## Encode to a CBOR array in binary form. This magic doesn't ## always work, some arguments may need to be explicitly ## converted with ``toCbor`` before passing. - str.writeCborArrayLen(args.len) + ?str.writeCborArrayLen(args.len) for x in args: - str.writeCbor(x) + ?str.writeCbor(x) -proc encode*[T](v: T): string = +proc encode*[T](v: T): ?!string = ## Encode an arbitrary value to CBOR binary representation. ## A wrapper over ``writeCbor``. let s = newStringStream() - s.writeCbor(v) - s.data + let res = s.writeCbor(v) + if res.isFailure: + return failure(res.error) + success(s.data) -proc toRaw*(n: CborNode): CborNode = +proc toRaw*(n: CborNode): ?!CborNode = ## Reduce a CborNode to a string of bytes. - if n.kind == cborRaw: n - else: CborNode(kind: cborRaw, raw: encode(n)) + if n.kind == cborRaw: + return success(n) + else: + without res =? encode(n), error: + return failure(error) + return success(CborNode(kind: cborRaw, raw: res)) -proc isSorted(n: CborNode): bool = +proc isSorted(n: CborNode): ?!bool = ## Check if the item is sorted correctly. var lastRaw = "" for key in n.map.keys: - let thisRaw = key.toRaw.raw + without res =? key.toRaw, error: + return failure(error.msg) + let thisRaw = res.raw if lastRaw != "": - if cmp(lastRaw, thisRaw) > 0: return false + if cmp(lastRaw, thisRaw) > 0: return success(false) lastRaw = thisRaw - true + success(true) -proc sort*(n: var CborNode) = +proc sort*(n: var CborNode): ?!void = ## Sort a CBOR map object. - var tmp = initOrderedTable[CborNode, CborNode](n.map.len.nextPowerOfTwo) - for key, val in n.map.mpairs: - tmp[key.toRaw] = move(val) - sort(tmp) do (x, y: tuple[k: CborNode; v: CborNode]) -> int: - result = cmp(x.k.raw, y.k.raw) - n.map = move tmp + try: + var tmp = initOrderedTable[CborNode, CborNode](n.map.len.nextPowerOfTwo) + for key, val in n.map.mpairs: + without res =? key.toRaw, error: + return failure(error) + tmp[res] = move(val) + sort(tmp) do (x, y: tuple[k: CborNode; v: CborNode]) -> int: + result = cmp(x.k.raw, y.k.raw) + n.map = move tmp + success() + except Exception as e: + return failure(e.msg) -proc writeCborHook*(str: Stream; dt: DateTime) = +proc writeCborHook*(str: Stream; dt: DateTime): ?!void = ## Write a `DateTime` using the tagged string representation ## defined in RCF7049 section 2.4.1. - writeCborTag(str, 0) - writeCbor(str, format(dt, timeFormat)) + ?writeCborTag(str, 0) + ?writeCbor(str, format(dt, timeFormat)) -proc writeCborHook*(str: Stream; t: Time) = +proc writeCborHook*(str: Stream; t: Time): ?!void = ## Write a `Time` using the tagged numerical representation ## defined in RCF7049 section 2.4.1. - writeCborTag(str, 1) - writeCbor(str, t.toUnix) + ?writeCborTag(str, 1) + ?writeCbor(str, t.toUnix) -func toCbor*(x: CborNode): CborNode = x +func toCbor*(x: CborNode): ?!CborNode = success(x) -func toCbor*(x: SomeInteger): CborNode = +func toCbor*(x: SomeInteger): ?!CborNode = if x > 0: - CborNode(kind: cborUnsigned, uint: x.uint64) + success(CborNode(kind: cborUnsigned, uint: x.uint64)) else: - CborNode(kind: cborNegative, int: x.int64) + success(CborNode(kind: cborNegative, int: x.int64)) -func toCbor*(x: openArray[byte]): CborNode = - CborNode(kind: cborBytes, bytes: @x) +func toCbor*(x: openArray[byte]): ?!CborNode = + success(CborNode(kind: cborBytes, bytes: @x)) -func toCbor*(x: string): CborNode = - CborNode(kind: cborText, text: x) +func toCbor*(x: string): ?!CborNode = + success(CborNode(kind: cborText, text: x)) -func toCbor*(x: openArray[CborNode]): CborNode = - CborNode(kind: cborArray, seq: @x) +func toCbor*(x: openArray[CborNode]): ?!CborNode = + success(CborNode(kind: cborArray, seq: @x)) -func toCbor*(pairs: openArray[(CborNode, CborNode)]): CborNode = - CborNode(kind: cborMap, map: pairs.toOrderedTable) +func toCbor*(pairs: openArray[(CborNode, CborNode)]): ?!CborNode = + try: + return success(CborNode(kind: cborMap, map: pairs.toOrderedTable)) + except Exception as e: + return failure(e.msg) -func toCbor*(tag: uint64; val: CborNode): CborNode = - result = toCbor(val) - result.tag = some(tag) +func toCbor*(tag: uint64; val: CborNode): ?!CborNode = + without res =? toCbor(val), error: + return failure(error.msg) + var cnode = res + cnode.tag = some(tag) + return success(cnode) -func toCbor*(x: bool): CborNode = +func toCbor*(x: bool): ?!CborNode = case x of false: - CborNode(kind: cborSimple, simple: 20) + success(CborNode(kind: cborSimple, simple: 20)) of true: - CborNode(kind: cborSimple, simple: 21) + success(CborNode(kind: cborSimple, simple: 21)) -func toCbor*(x: SomeFloat): CborNode = - CborNode(kind: cborFloat, float: x.float64) +func toCbor*(x: SomeFloat): ?!CborNode = + success(CborNode(kind: cborFloat, float: x.float64)) -func toCbor*(x: pointer): CborNode = +func toCbor*(x: pointer): ?!CborNode = ## A hack to produce a CBOR null item. assert(x.isNil) - CborNode(kind: cborSimple, simple: 22) + if not x.isNil: + return failure("pointer is not nil") + success(CborNode(kind: cborSimple, simple: 22)) func initCborBytes*[T: char|byte](buf: openArray[T]): CborNode = ## Create a CBOR byte string from `buf`. @@ -359,11 +399,4 @@ func initCbor*(items: varargs[CborNode, toCbor]): CborNode = ## Initialize a CBOR arrary. CborNode(kind: cborArray, seq: @items) -template initCborOther*(x: untyped): CborNode = - ## Initialize a ``CborNode`` from a type where ``toCbor`` is not implemented. - ## This encodes ``x`` to binary using ``writeCbor``, so - ## ``$(initCborOther(x))`` will incur an encode and decode roundtrip. - let s = newStringStream() - s.writeCbor(x) - CborNode(kind: cborRaw, raw: s.data) diff --git a/tests/cbor/testObjects.nim b/tests/cbor/testObjects.nim index 9c1bb2e..4b0deed 100644 --- a/tests/cbor/testObjects.nim +++ b/tests/cbor/testObjects.nim @@ -21,11 +21,6 @@ type point: CustomPoint color: CustomColor - Person = object - name: string - age: int - isActive: bool - Inner = object s: string nums: seq[int] @@ -134,12 +129,22 @@ suite "CBOR deserialization": refInner: refObj ) + # Serialize to CBOR with encode + without encodedStr =? encode(original), error: + fail() + # Serialize to CBOR let stream = newStringStream() - stream.writeCbor(original) + if stream.writeCbor(original).isFailure: + fail() + let cborData = stream.data + # Check that both serialized forms are equal + check cborData == encodedStr + # Parse CBOR back to CborNode let parseResult = parseCbor(cborData) + check parseResult.isSuccess let node = parseResult.tryValue From eadc7e2d168f1198373151197dc30ce8b8447ee1 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Thu, 22 May 2025 15:39:43 +0530 Subject: [PATCH 05/20] chore: fix test cases --- serde/cbor/deserializer.nim | 30 ++++++++++++---------------- serde/cbor/serializer.nim | 8 +++++--- tests/cbor/testprimitives.nim | 37 +++++++++++++++++++---------------- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 3d406f9..3d08e41 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -88,9 +88,9 @@ proc nextUInt*(c: var CborParser): ?!BiggestUInt = if c.kind != CborEventKind.cborPositive: return failure(newCborError("Expected positive integer, got " & $c.kind)) let val = c.intVal.BiggestUInt - let nextRes = c.next() - if nextRes.isFailure: - return failure(nextRes.error) + + ?c.next() + return success(val) @@ -105,9 +105,7 @@ proc nextInt*(c: var CborParser): ?!BiggestInt = else: return failure(newCborError("Expected integer, got " & $c.kind)) - let nextRes = c.next() - if nextRes.isFailure: - return failure(nextRes.error) + ?c.next() return success(val) @@ -126,9 +124,7 @@ proc nextFloat*(c: var CborParser): ?!float64 = else: discard - let nextRes = c.next() - if nextRes.isFailure: - return failure(nextRes.error) + ?c.next() return success(val) @@ -498,31 +494,31 @@ proc parseTime(n: CborNode): Time = else: assert false -proc fromCborHook*(v: var DateTime; n: CborNode): bool = +proc fromCborHook*(v: var DateTime; n: CborNode): ?!void = ## Parse a `DateTime` from the tagged string representation ## defined in RCF7049 section 2.4.1. if n.tag.isSome: try: if n.tag.get == 0 and n.kind == cborText: v = parseDateText(n) - result = true + return success() elif n.tag.get == 1 and n.kind in {cborUnsigned, cborNegative, cborFloat}: v = parseTime(n).utc - result = true - except ValueError: discard + return success() + except ValueError as e: return failure(e) -proc fromCborHook*(v: var Time; n: CborNode): bool = +proc fromCborHook*(v: var Time; n: CborNode): ?!void = ## Parse a `Time` from the tagged string representation ## defined in RCF7049 section 2.4.1. if n.tag.isSome: try: if n.tag.get == 0 and n.kind == cborText: v = parseDateText(n).toTime - result = true + return success() elif n.tag.get == 1 and n.kind in {cborUnsigned, cborNegative, cborFloat}: v = parseTime(n) - result = true - except ValueError: discard + return success() + except ValueError as e: return failure(e) func isTagged*(n: CborNode): bool = ## Check if a CBOR item has a tag. diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index 4165547..294b2ce 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -119,7 +119,7 @@ proc writeCbor*[T](str: Stream; v: T): ?!void = try: when T is CborNode: if v.tag.isSome: - return str.writeCborTag(v.tag.get) + ?str.writeCborTag(v.tag.get) case v.kind: of cborUnsigned: return str.writeCbor(v.uint) @@ -158,7 +158,7 @@ proc writeCbor*[T](str: Stream; v: T): ?!void = of cborRaw: str.write(v.raw) elif compiles(writeCborHook(str, v)): - writeCborHook(str, v) + ?writeCborHook(str, v) elif T is SomeUnsignedInt: ?str.writeInitial(0, v) elif T is SomeSignedInt: @@ -231,7 +231,6 @@ proc writeCbor*[T](str: Stream; v: T): ?!void = var be: float64 swapEndian64 be.addr, v.unsafeAddr str.write be - return success() of fcZero: str.write initialByte(7, 25) @@ -262,6 +261,7 @@ proc writeCborArray*(str: Stream; args: varargs[CborNode, toCbor]): ?!void = ?str.writeCborArrayLen(args.len) for x in args: ?str.writeCbor(x) + success() proc encode*[T](v: T): ?!string = ## Encode an arbitrary value to CBOR binary representation. @@ -313,12 +313,14 @@ proc writeCborHook*(str: Stream; dt: DateTime): ?!void = ## defined in RCF7049 section 2.4.1. ?writeCborTag(str, 0) ?writeCbor(str, format(dt, timeFormat)) + success() proc writeCborHook*(str: Stream; t: Time): ?!void = ## Write a `Time` using the tagged numerical representation ## defined in RCF7049 section 2.4.1. ?writeCborTag(str, 1) ?writeCbor(str, t.toUnix) + success() func toCbor*(x: CborNode): ?!CborNode = success(x) diff --git a/tests/cbor/testprimitives.nim b/tests/cbor/testprimitives.nim index 0cf988c..a3e4ae7 100644 --- a/tests/cbor/testprimitives.nim +++ b/tests/cbor/testprimitives.nim @@ -54,7 +54,8 @@ suite "roundtrip": without c =? parseCbor(controlCbor), error: fail() test name: - let testCbor = encode(c) + without testCbor =? encode(c), error: + fail() if controlCbor != testCbor: let testB64 = base64.encode(testCbor) check(controlB64 == testB64) @@ -62,42 +63,44 @@ suite "roundtrip": suite "hooks": test "DateTime": let dt = now() - var - bin = encode(dt) + + without bin =? encode(dt), error: + fail() without node =? parseCbor(bin), error: fail() check(node.text == $dt) test "Time": let t = now().toTime var - bin = encode(t) + bin = encode(t).tryValue without node =? parseCbor(bin), error: fail() check(node.getInt == t.toUnix) test "tag": - var c = toCbor("foo") + var c = toCbor("foo").tryValue c.tag = some(99'u64) check c.tag == some(99'u64) test "sorting": var map = initCborMap() var keys = @[ - toCbor(10), - toCbor(100), - toCbor(-1), - toCbor("z"), - toCbor("aa"), - toCbor([toCbor(100)]), - toCbor([toCbor(-1)]), - toCbor(false), + toCbor(10).tryValue, + toCbor(100).tryValue, + toCbor(-1).tryValue, + toCbor("z").tryValue, + toCbor("aa").tryValue, + toCbor([toCbor(100).tryValue]).tryValue, + toCbor([toCbor(-1).tryValue]).tryValue, + toCbor(false).tryValue, ] shuffle(keys) - for k in keys: map[k] = toCbor(0) - check not map.isSorted - sort(map) - check map.isSorted + for k in keys: map[k] = toCbor(0).tryValue + + check not map.isSorted.tryValue + check sort(map).isSuccess + check map.isSorted.tryValue test "invalid wire type": var intValue: int From 84943dbed912f26482a0d2bdc1a5586a8828e4bc Mon Sep 17 00:00:00 2001 From: munna0908 Date: Thu, 22 May 2025 16:39:37 +0530 Subject: [PATCH 06/20] update license info --- serde/cbor/deserializer.nim | 13 +++---------- serde/cbor/errors.nim | 4 +++- serde/cbor/helpers.nim | 13 +++++++++++++ serde/cbor/jsonhook.nim | 3 +++ serde/cbor/serializer.nim | 5 ++++- serde/cbor/types.nim | 3 +++ tests/cbor/testprimitives.nim | 6 ++++-- 7 files changed, 33 insertions(+), 14 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 3d08e41..714f97b 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -1,3 +1,6 @@ +# This file is a modified version of Emery Hemingway’s CBOR library for Nim, +# originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. + import std/[math, streams, options, tables, strutils, times, typetraits] import ./types import ./helpers @@ -134,16 +137,6 @@ func bytesLen*(c: CborParser): ?!int = return failure(newCborError("Expected bytes, got " & $c.kind)) return success(c.intVal.int) -template tryNext(c: var CborParser) = - let nextRes = c.next() - if nextRes.isFailure: - return failure(nextRes.error) - -template trySkip(c: var CborParser) = - let skipRes = c.skipNode() - if skipRes.isFailure: - return failure(skipRes.error) - proc nextBytes*(c: var CborParser; buf: var openArray[byte]): ?!void = ## Read the bytes that the parser is positioned on and advance. try: diff --git a/serde/cbor/errors.nim b/serde/cbor/errors.nim index f375a1a..db6cdc5 100644 --- a/serde/cbor/errors.nim +++ b/serde/cbor/errors.nim @@ -1,8 +1,10 @@ +# This file is a modified version of Emery Hemingway’s CBOR library for Nim, +# originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. + import ../utils/types import ./types import std/sets - proc newUnexpectedKindError*( expectedType: type, expectedKinds: string, cbor: CborNode ): ref UnexpectedKindError = diff --git a/serde/cbor/helpers.nim b/serde/cbor/helpers.nim index 8450534..4e32ee5 100644 --- a/serde/cbor/helpers.nim +++ b/serde/cbor/helpers.nim @@ -1,8 +1,21 @@ +# This file is a modified version of Emery Hemingway’s CBOR library for Nim, +# originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. import ./types import ./errors from macros import newDotExpr, newIdentNode, strVal + +template tryNext*(c: var CborParser) = + let nextRes = c.next() + if nextRes.isFailure: + return failure(nextRes.error) + +template trySkip*(c: var CborParser) = + let skipRes = c.skipNode() + if skipRes.isFailure: + return failure(skipRes.error) + template exceptCborKind*(expectedType: type, expectedKinds: set[CborNodeKind], cbor: CborNode) = if cbor.kind notin expectedKinds: diff --git a/serde/cbor/jsonhook.nim b/serde/cbor/jsonhook.nim index 21e7a7a..6486225 100644 --- a/serde/cbor/jsonhook.nim +++ b/serde/cbor/jsonhook.nim @@ -1,3 +1,6 @@ +# This file is a modified version of Emery Hemingway’s CBOR library for Nim, +# originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. + import std/[base64, tables] import ../json/stdjson import ./types diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index 294b2ce..d605b85 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -1,4 +1,7 @@ -import std/[streams, options, tables, typetraits, math, endians, times, base64] +# This file is a modified version of Emery Hemingway’s CBOR library for Nim, +# originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. + +import std/[streams, options, tables, typetraits, math, endians, times] import pkg/questionable import pkg/questionable/results import ../utils/errors diff --git a/serde/cbor/types.nim b/serde/cbor/types.nim index 8da53d5..560cefa 100644 --- a/serde/cbor/types.nim +++ b/serde/cbor/types.nim @@ -1,3 +1,6 @@ +# This file is a modified version of Emery Hemingway’s CBOR library for Nim, +# originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. + import std/[streams, tables, options, hashes, times] const timeFormat* = initTimeFormat "yyyy-MM-dd'T'HH:mm:sszzz" diff --git a/tests/cbor/testprimitives.nim b/tests/cbor/testprimitives.nim index a3e4ae7..b555d37 100644 --- a/tests/cbor/testprimitives.nim +++ b/tests/cbor/testprimitives.nim @@ -1,5 +1,7 @@ -import - std/[base64, os, random, times, json, unittest] +# This file is a modified version of Emery Hemingway’s CBOR library for Nim, +# originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. + +import std/[base64, os, random, times, json, unittest] import pkg/serde/cbor import pkg/questionable import pkg/questionable/results From 587c834c0ef0797b05b1e0fc407e45b6015df4c8 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Thu, 22 May 2025 16:50:09 +0530 Subject: [PATCH 07/20] chore: cosmetic changes --- serde/cbor/deserializer.nim | 3 --- 1 file changed, 3 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 714f97b..92928d1 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -93,7 +93,6 @@ proc nextUInt*(c: var CborParser): ?!BiggestUInt = let val = c.intVal.BiggestUInt ?c.next() - return success(val) @@ -109,7 +108,6 @@ proc nextInt*(c: var CborParser): ?!BiggestInt = return failure(newCborError("Expected integer, got " & $c.kind)) ?c.next() - return success(val) proc nextFloat*(c: var CborParser): ?!float64 = @@ -128,7 +126,6 @@ proc nextFloat*(c: var CborParser): ?!float64 = discard ?c.next() - return success(val) func bytesLen*(c: CborParser): ?!int = From d3c3774391533826ab0ded3d7432ed52aef1200b Mon Sep 17 00:00:00 2001 From: munna0908 Date: Thu, 22 May 2025 17:23:20 +0530 Subject: [PATCH 08/20] fix test filename --- tests/cbor/testObjects.nim | 1 - tests/cbor/{testprimitives.nim => testPrimitives.nim} | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) rename tests/cbor/{testprimitives.nim => testPrimitives.nim} (100%) diff --git a/tests/cbor/testObjects.nim b/tests/cbor/testObjects.nim index 4b0deed..2f8415e 100644 --- a/tests/cbor/testObjects.nim +++ b/tests/cbor/testObjects.nim @@ -1,4 +1,3 @@ -# File: /Users/rahul/Work/repos/nim-serde/tests/cbor_questionable.nim import std/unittest import std/options diff --git a/tests/cbor/testprimitives.nim b/tests/cbor/testPrimitives.nim similarity index 100% rename from tests/cbor/testprimitives.nim rename to tests/cbor/testPrimitives.nim index b555d37..2741390 100644 --- a/tests/cbor/testprimitives.nim +++ b/tests/cbor/testPrimitives.nim @@ -98,8 +98,8 @@ test "sorting": toCbor(false).tryValue, ] shuffle(keys) - for k in keys: map[k] = toCbor(0).tryValue + for k in keys: map[k] = toCbor(0).tryValue check not map.isSorted.tryValue check sort(map).isSuccess check map.isSorted.tryValue From 056a78e95f6b5e8a3303d60d73836f328f57f569 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Wed, 28 May 2025 11:49:50 +0530 Subject: [PATCH 09/20] refactor: reimplement CBOR deserialization with type-based fromCbor interface --- serde/cbor/deserializer.nim | 220 +++++++++++++++++++++++++--------- serde/cbor/helpers.nim | 24 +--- serde/cbor/jsonhook.nim | 10 +- serde/cbor/serializer.nim | 14 ++- serde/cbor/types.nim | 3 +- serde/utils/pragmas.nim | 15 ++- tests/cbor/testObjects.nim | 28 ++--- tests/cbor/testPrimitives.nim | 2 +- 8 files changed, 210 insertions(+), 106 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 92928d1..0b3432d 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -1,16 +1,20 @@ # This file is a modified version of Emery Hemingway’s CBOR library for Nim, # originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. -import std/[math, streams, options, tables, strutils, times, typetraits] +import std/[math, streams, options, tables, strutils, times, typetraits, macros] import ./types import ./helpers -import ../utils/types +import ../utils/types as utilsTypes +import ../utils/pragmas import ./errors import pkg/questionable import pkg/questionable/results export results export types +export pragmas +export utilsTypes + {.push raises: [].} @@ -95,7 +99,6 @@ proc nextUInt*(c: var CborParser): ?!BiggestUInt = ?c.next() return success(val) - proc nextInt*(c: var CborParser): ?!BiggestInt = ## Parse the integer value that the parser is positioned on. var val: BiggestInt @@ -146,7 +149,7 @@ proc nextBytes*(c: var CborParser; buf: var openArray[byte]): ?!void = let n = c.s.readData(buf[0].addr, buf.len) if n != buf.len: return failure(newCborError("truncated read of CBOR data")) - tryNext(c) + ?c.next() success() except OSError as e: return failure(e.msg) @@ -178,14 +181,13 @@ proc nextText*(c: var CborParser; buf: var string): ?!void = let n = c.s.readData(buf[0].addr, buf.len) if n != buf.len: return failure(newCborError("truncated read of CBOR data")) - tryNext(c) + ?c.next() success() except IOError as e: return failure(e.msg) except OSError as e: return failure(e.msg) - proc nextText*(c: var CborParser): ?!string = ## Read the text that the parser is positioned on into a string and advance. var buf: string @@ -195,20 +197,26 @@ proc nextText*(c: var CborParser): ?!string = return success(buf) -func arrayLen*(c: CborParser): int = +func arrayLen*(c: CborParser): ?!int = ## Return the length of the array that the parser is positioned on. - assert(c.kind == CborEventKind.cborArray, $c.kind) - c.intVal.int + if c.kind != CborEventKind.cborArray: + return failure(newCborError("Expected array, got " & $c.kind)) -func mapLen*(c: CborParser): int = + return success(c.intVal.int) + +func mapLen*(c: CborParser): ?!int = ## Return the length of the map that the parser is positioned on. - assert(c.kind == CborEventKind.cborMap, $c.kind) - c.intVal.int + if c.kind != CborEventKind.cborMap: + return failure(newCborError("Expected map, got " & $c.kind)) -func tag*(c: CborParser): uint64 = + return success(c.intVal.int) + +func tag*(c: CborParser): ?!uint64 = ## Return the tag value the parser is positioned on. - assert(c.kind == CborEventKind.cborTag, $c.kind) - c.intVal + if c.kind != CborEventKind.cborTag: + return failure(newCborError("Expected tag, got " & $c.kind)) + + return success(c.intVal) proc skipNode*(c: var CborParser): ?!void = ## Skip the item the parser is positioned on. @@ -221,7 +229,7 @@ proc skipNode*(c: var CborParser): ?!void = return c.next() of CborEventKind.cborBytes, CborEventKind.cborText: if c.isIndefinite: - tryNext(c) + ?c.next() while c.kind != CborEventKind.cborBreak: if c.kind != CborEventKind.cborBytes: return failure(newCborError("expected bytes, got " & $c.kind)) @@ -232,28 +240,28 @@ proc skipNode*(c: var CborParser): ?!void = return c.next() of CborEventKind.cborArray: if c.isIndefinite: - tryNext(c) + ?c.next() while c.kind != CborEventKind.cborBreak: - trySkip(c) + ?c.skipNode() return c.next() else: let len = c.intVal - tryNext(c) + ?c.next() for i in 1..len: - trySkip(c) + ?c.skipNode() of CborEventKind.cborMap: let mapLen = c.intVal.int if c.isIndefinite: - tryNext(c) + ?c.next() while c.kind != CborEventKind.cborBreak: - trySkip(c) + ?c.skipNode() return c.next() else: - tryNext(c) + ?c.next() for _ in 1 .. mapLen: - trySkip(c) + ?c.skipNode() of CborEventKind.cborTag: - tryNext(c) + ?c.next() return c.skipNode() of CborEventKind.cborFloat: without f =? c.nextFloat(), error: @@ -279,14 +287,14 @@ proc nextNode*(c: var CborParser): ?!CborNode = return failure(newCborError("end of CBOR stream")) of CborEventKind.cborPositive: next = CborNode(kind: cborUnsigned, uint: c.intVal) - tryNext(c) + ?c.next() of CborEventKind.cborNegative: next = CborNode(kind: cborNegative, int: -1 - c.intVal.int64) - tryNext(c) + ?c.next() of CborEventKind.cborBytes: if c.isIndefinite: next = CborNode(kind: cborBytes, bytes: newSeq[byte]()) - tryNext(c) + ?c.next() while c.kind != CborEventKind.cborBreak: if c.kind != CborEventKind.cborBytes: return failure(newCborError("Expected bytes, got " & $c.kind)) @@ -297,7 +305,7 @@ proc nextNode*(c: var CborParser): ?!CborNode = let n = c.s.readData(next.bytes[pos].addr, chunkLen) if n != chunkLen: return failure(newCborError("truncated read of CBOR data")) - tryNext(c) + ?c.next() else: without rawBytes =? c.nextBytes(), error: return failure(error) @@ -305,7 +313,7 @@ proc nextNode*(c: var CborParser): ?!CborNode = of CborEventKind.cborText: if c.isIndefinite: next = CborNode(kind: cborText, text: "") - tryNext(c) + ?c.next() while c.kind != CborEventKind.cborBreak: if c.kind != CborEventKind.cborText: return failure(newCborError("Expected text, got " & $c.kind)) @@ -316,8 +324,8 @@ proc nextNode*(c: var CborParser): ?!CborNode = let n = c.s.readData(next.text[pos].addr, chunkLen) if n != chunkLen: return failure(newCborError("truncated read of CBOR data")) - tryNext(c) - tryNext(c) + ?c.next() + ?c.next() else: without text =? c.nextText(), error: return failure(error) @@ -325,14 +333,14 @@ proc nextNode*(c: var CborParser): ?!CborNode = of CborEventKind.cborArray: next = CborNode(kind: cborArray, seq: newSeq[CborNode](c.intVal)) if c.isIndefinite: - tryNext(c) + ?c.next() while c.kind != CborEventKind.cborBreak: without node =? c.nextNode(), error: return failure(error) next.seq.add(node) - tryNext(c) + ?c.next() else: - tryNext(c) + ?c.next() for i in 0..next.seq.high: without node =? c.nextNode(), error: return failure(error) @@ -342,16 +350,16 @@ proc nextNode*(c: var CborParser): ?!CborNode = next = CborNode(kind: cborMap, map: initOrderedTable[CborNode, CborNode]( mapLen.nextPowerOfTwo)) if c.isIndefinite: - tryNext(c) + ?c.next() while c.kind != CborEventKind.cborBreak: without key =? c.nextNode(), error: return failure(error) without val =? c.nextNode(), error: return failure(error) next.map[key] = val - tryNext(c) + ?c.next() else: - tryNext(c) + ?c.next() for _ in 1 .. mapLen: without key =? c.nextNode(), error: return failure(error) @@ -360,7 +368,7 @@ proc nextNode*(c: var CborParser): ?!CborNode = next.map[key] = val of CborEventKind.cborTag: let tag = c.intVal - tryNext(c) + ?c.next() without node =? c.nextNode(), error: return failure(error) next = node @@ -371,7 +379,7 @@ proc nextNode*(c: var CborParser): ?!CborNode = next = CborNode(kind: cborSimple, simple: c.intVal.uint8) else: next = CborNode(kind: cborSimple, simple: c.minor) - tryNext(c) + ?c.next() of CborEventKind.cborFloat: without f =? c.nextFloat(), error: return failure(error) @@ -383,15 +391,17 @@ proc nextNode*(c: var CborParser): ?!CborNode = return failure(e.msg) except IOError as e: return failure(e.msg) - except Exception as e: + except CatchableError as e: return failure(e.msg) + except Exception as e: + raise newException(Defect, e.msg, e) proc readCbor*(s: Stream): ?!CborNode = ## Parse a stream into a CBOR object. var parser: CborParser parser.open(s) - tryNext(parser) + ?parser.next() parser.nextNode() proc parseCbor*(s: string): ?!CborNode = @@ -473,7 +483,7 @@ proc getInt*(n: CborNode; default: int = 0): int = else: default proc parseDateText(n: CborNode): DateTime {.raises: [TimeParseError].} = - parse(n.text, timeFormat) + parse(n.text, dateTimeFormat) proc parseTime(n: CborNode): Time = case n.kind @@ -557,7 +567,6 @@ proc getSigned*(n: CborNode; default: int64 = 0): int64 = of cborNegative: n.int else: default - func getFloat*(n: CborNode; default = 0.0): float = ## Get the floating-poing value of a ``CborNode`` or a fallback. if n.kind == cborFloat: @@ -565,12 +574,69 @@ func getFloat*(n: CborNode; default = 0.0): float = else: default +proc fromCbor*[T: distinct](_: type T; n: CborNode): ?!T = + success T(?T.distinctBase.fromCbor(n)) + +proc fromCbor*[T: SomeUnsignedInt](_: type T; n: CborNode): ?!T = + expectCborKind(T, {cborUnsigned}, n) + var v = T(n.uint) + if v.BiggestUInt == n.uint: + return success(v) + else: + return failure(newCborError("Value overflow for unsigned integer")) + +proc fromCbor*[T: SomeSignedInt](_: type T; n: CborNode): ?!T = + expectCborKind(T, {cborUnsigned, cborNegative}, n) + if n.kind == cborUnsigned: + var v = T(n.uint) + if v.BiggestUInt == n.uint: + return success(v) + else: + return failure(newCborError("Value overflow for signed integer")) + elif n.kind == cborNegative: + var v = T(n.int) + if v.BiggestInt == n.int: + return success(v) + else: + return failure(newCborError("Value overflow for signed integer")) + +proc fromCbor*[T: SomeFloat](_: type T; n: CborNode): ?!T = + expectCborKind(T, {cborFloat}, n) + return success(T(n.float)) + +proc fromCbor*(_: type seq[byte]; n: CborNode): ?!seq[byte] = + expectCborKind(seq[byte], cborBytes, n) + return success(n.bytes) + +proc fromCbor*(_: type string; n: CborNode): ?!string = + expectCborKind(string, cborText, n) + return success(n.text) + +proc fromCbor*(_: type bool; n: CborNode): ?!bool = + if not n.isBool: + return failure(newCborError("Expected boolean, got " & $n.kind)) + return success(n.getBool) + +proc fromCbor*[T](_: type seq[T]; n: CborNode): ?!seq[T] = + expectCborKind(seq[T], cborArray, n) + var arr = newSeq[T](n.seq.len) + for i, elem in n.seq: + arr[i] = ?T.fromCbor(elem) + success arr + +proc fromCbor*[T: tuple](_: type T; n: CborNode): ?!T = + expectCborKind(T, cborArray, n) + var res = T.default + if n.seq.len != T.tupleLen: + return failure(newCborError("Expected tuple of length " & $T.tupleLen)) + var i: int + for f in fields(res): + f = ?typeof(f).fromCbor(n.seq[i]) + inc i + + success res proc fromCbor*[T](v: var T; n: CborNode): ?!void = - ## Return a Result containing the value if `v` can be converted from a given `CborNode`, - ## or an error if conversion fails. - ## Can be extended and overriden with `fromCborHook(v: var T; n: CborNode)` - ## for specific types of `T`. try: when T is CborNode: v = n @@ -580,14 +646,14 @@ proc fromCbor*[T](v: var T; n: CborNode): ?!void = elif T is distinct: return fromCbor(distinctBase v, n) elif T is SomeUnsignedInt: - exceptCborKind(T, {cborUnsigned}, n) + expectCborKind(T, {cborUnsigned}, n) v = T n.uint if v.BiggestUInt == n.uint: return success() else: return failure(newCborError("Value overflow for unsigned integer")) elif T is SomeSignedInt: - exceptCborKind(T, {cborUnsigned, cborNegative}, n) + expectCborKind(T, {cborUnsigned, cborNegative}, n) if n.kind == cborUnsigned: v = T n.uint if v.BiggestUInt == n.uint: @@ -606,19 +672,19 @@ proc fromCbor*[T](v: var T; n: CborNode): ?!void = v = n.getBool return success() elif T is SomeFloat: - exceptCborKind(T, {cborFloat}, n) + expectCborKind(T, {cborFloat}, n) v = T n.float return success() elif T is seq[byte]: - exceptCborKind(T, {cborBytes}, n) + expectCborKind(T, {cborBytes}, n) v = n.bytes return success() elif T is string: - exceptCborKind(T, {cborText}, n) + expectCborKind(T, {cborText}, n) v = n.text return success() elif T is seq: - exceptCborKind(T, {cborArray}, n) + expectCborKind(T, {cborArray}, n) v.setLen n.seq.len for i, e in n.seq: let itemResult = fromCbor(v[i], e) @@ -627,7 +693,7 @@ proc fromCbor*[T](v: var T; n: CborNode): ?!void = return failure(itemResult.error) return success() elif T is tuple: - exceptCborKind(T, {cborArray}, n) + expectCborKind(T, {cborArray}, n) if n.seq.len != T.tupleLen: return failure(newCborError("Expected tuple of length " & $T.tupleLen)) var i: int @@ -645,7 +711,7 @@ proc fromCbor*[T](v: var T; n: CborNode): ?!void = if isNil(v): new(v) return fromCbor(v[], n) elif T is object: - exceptCborKind(T, {cborMap}, n) + expectCborKind(T, {cborMap}, n) var i: int key = CborNode(kind: cborText) @@ -664,5 +730,45 @@ proc fromCbor*[T](v: var T; n: CborNode): ?!void = return failure(newCborError("Extra fields in map")) else: return failure(newCborError("Unsupported type: " & $T)) + except CatchableError as e: + return failure newCborError(e.msg) except Exception as e: - return failure(newCborError(e.msg)) + raise newException(Defect, e.msg, e) + +proc fromCbor*[T: ref object or object](_: type T; n: CborNode): ?!T = + when T is CborNode: + return success T(n) + + expectCborKind(T, {cborMap}, n) + + var res = + when type(T) is ref: + T.new() + else: + T.default + + try: + var + i: int + key = CborNode(kind: cborText) + for name, value in fieldPairs( + when type(T) is ref: + res[] + else: + res + ): + key.text = name + + if not n.map.hasKey key: + return failure(newCborError("Missing field: " & name)) + else: + value = ?typeof(value).fromCbor(n.map[key]) + inc i + if i == n.map.len: + return success(res) + else: + return failure(newCborError("Extra fields in map")) + except CatchableError as e: + return failure newCborError(e.msg) + except Exception as e: + raise newException(Defect, e.msg, e) diff --git a/serde/cbor/helpers.nim b/serde/cbor/helpers.nim index 4e32ee5..4332dab 100644 --- a/serde/cbor/helpers.nim +++ b/serde/cbor/helpers.nim @@ -5,40 +5,28 @@ import ./types import ./errors from macros import newDotExpr, newIdentNode, strVal - -template tryNext*(c: var CborParser) = - let nextRes = c.next() - if nextRes.isFailure: - return failure(nextRes.error) - -template trySkip*(c: var CborParser) = - let skipRes = c.skipNode() - if skipRes.isFailure: - return failure(skipRes.error) - -template exceptCborKind*(expectedType: type, expectedKinds: set[CborNodeKind], +template expectCborKind*(expectedType: type, expectedKinds: set[CborNodeKind], cbor: CborNode) = if cbor.kind notin expectedKinds: return failure(newUnexpectedKindError(expectedType, expectedKinds, cbor)) -template exceptCborKind*(expectedType: type, expectedKind: CborNodeKind, +template expectCborKind*(expectedType: type, expectedKind: CborNodeKind, cbor: CborNode) = - exceptCborKind(expectedType, {expectedKind}, cbor) + expectCborKind(expectedType, {expectedKind}, cbor) -template exceptCborKind*(expectedType: type, expectedKinds: set[CborEventKind], +template expectCborKind*(expectedType: type, expectedKinds: set[CborEventKind], cbor: CborNode) = if cbor.kind notin expectedKinds: return failure(newUnexpectedKindError(expectedType, expectedKinds, cbor)) -template exceptCborKind*(expectedType: type, expectedKind: CborEventKind, +template expectCborKind*(expectedType: type, expectedKind: CborEventKind, cbor: CborNode) = - exceptCborKind(expectedType, {expectedKind}, cbor) + expectCborKind(expectedType, {expectedKind}, cbor) macro dot*(obj: object, fld: string): untyped = ## Turn ``obj.dot("fld")`` into ``obj.fld``. newDotExpr(obj, newIdentNode(fld.strVal)) - func floatSingle*(half: uint16): float32 = ## Convert a 16-bit float to 32-bits. func ldexp(x: float64; exponent: int): float64 {.importc: "ldexp", diff --git a/serde/cbor/jsonhook.nim b/serde/cbor/jsonhook.nim index 6486225..faccd1a 100644 --- a/serde/cbor/jsonhook.nim +++ b/serde/cbor/jsonhook.nim @@ -7,7 +7,7 @@ import ./types import ./errors import ./deserializer -proc toJsonHook*(n: CborNode): JsonNode = +proc toJson*(n: CborNode): JsonNode = case n.kind: of cborUnsigned: newJInt n.uint.BiggestInt @@ -20,15 +20,15 @@ proc toJsonHook*(n: CborNode): JsonNode = of cborArray: let a = newJArray() for e in n.seq.items: - a.add(e.toJsonHook) + a.add(e.toJson) a of cborMap: let o = newJObject() for k, v in n.map.pairs: if k.kind == cborText: - o[k.text] = v.toJsonHook + o[k.text] = v.toJson else: - o[$k] = v.toJsonHook + o[$k] = v.toJson o of cborTag: nil of cborSimple: @@ -42,4 +42,4 @@ proc toJsonHook*(n: CborNode): JsonNode = of cborRaw: without parsed =? parseCbor(n.raw), error: raise newCborError(error.msg) - toJsonHook(parsed) + toJson(parsed) diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index d605b85..2c0b014 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -303,19 +303,22 @@ proc sort*(n: var CborNode): ?!void = for key, val in n.map.mpairs: without res =? key.toRaw, error: return failure(error) - tmp[res] = move(val) + if tmp.hasKey(res): + tmp[res] = move(val) sort(tmp) do (x, y: tuple[k: CborNode; v: CborNode]) -> int: result = cmp(x.k.raw, y.k.raw) n.map = move tmp success() - except Exception as e: + except CatchableError as e: return failure(e.msg) + except Exception as e: + raise newException(Defect, e.msg, e) proc writeCborHook*(str: Stream; dt: DateTime): ?!void = ## Write a `DateTime` using the tagged string representation ## defined in RCF7049 section 2.4.1. ?writeCborTag(str, 0) - ?writeCbor(str, format(dt, timeFormat)) + ?writeCbor(str, format(dt, dateTimeFormat)) success() proc writeCborHook*(str: Stream; t: Time): ?!void = @@ -345,8 +348,10 @@ func toCbor*(x: openArray[CborNode]): ?!CborNode = func toCbor*(pairs: openArray[(CborNode, CborNode)]): ?!CborNode = try: return success(CborNode(kind: cborMap, map: pairs.toOrderedTable)) - except Exception as e: + except CatchableError as e: return failure(e.msg) + except Exception as e: + raise newException(Defect, e.msg, e) func toCbor*(tag: uint64; val: CborNode): ?!CborNode = without res =? toCbor(val), error: @@ -367,7 +372,6 @@ func toCbor*(x: SomeFloat): ?!CborNode = func toCbor*(x: pointer): ?!CborNode = ## A hack to produce a CBOR null item. - assert(x.isNil) if not x.isNil: return failure("pointer is not nil") success(CborNode(kind: cborSimple, simple: 22)) diff --git a/serde/cbor/types.nim b/serde/cbor/types.nim index 560cefa..c7e3214 100644 --- a/serde/cbor/types.nim +++ b/serde/cbor/types.nim @@ -3,7 +3,8 @@ import std/[streams, tables, options, hashes, times] -const timeFormat* = initTimeFormat "yyyy-MM-dd'T'HH:mm:sszzz" +# This format is defined in RCF8949 section 3.4.1. +const dateTimeFormat* = initTimeFormat "yyyy-MM-dd'T'HH:mm:sszzz" const PositiveMajor* = 0'u8 diff --git a/serde/utils/pragmas.nim b/serde/utils/pragmas.nim index e6dede9..69d243e 100644 --- a/serde/utils/pragmas.nim +++ b/serde/utils/pragmas.nim @@ -6,8 +6,10 @@ export types {.push raises: [].} -template serialize*(key = "", ignore = false, mode = SerdeMode.OptOut) {.pragma.} -template deserialize*(key = "", ignore = false, mode = SerdeMode.OptOut) {.pragma.} +template serialize*(key = "", ignore = false, + mode = SerdeMode.OptOut) {.pragma.} +template deserialize*(key = "", ignore = false, + mode = SerdeMode.OptOut) {.pragma.} proc isDefault[T](paramValue: T): bool {.compileTime.} = when T is SerdeMode: @@ -23,7 +25,8 @@ template expectMissingPragmaParam*(value, pragma, name, msg) = if paramName == name and not paramValue.isDefault: raiseAssert(msg) -template getSerdeFieldOptions*(pragma, fieldName, fieldValue): SerdeFieldOptions = +template getSerdeFieldOptions*(pragma, fieldName, + fieldValue): SerdeFieldOptions = var opts = SerdeFieldOptions(key: fieldName, ignore: false) when fieldValue.hasCustomPragma(pragma): fieldValue.expectMissingPragmaParam( @@ -43,12 +46,14 @@ template getSerdeMode*(T, pragma): SerdeMode = T.expectMissingPragmaParam( pragma, "key", - "Cannot set " & astToStr(pragma) & " 'key' on '" & $T & "' type definition.", + "Cannot set " & astToStr(pragma) & " 'key' on '" & $T & + "' type definition.", ) T.expectMissingPragmaParam( pragma, "ignore", - "Cannot set " & astToStr(pragma) & " 'ignore' on '" & $T & "' type definition.", + "Cannot set " & astToStr(pragma) & " 'ignore' on '" & $T & + "' type definition.", ) let (_, _, mode) = T.getCustomPragmaVal(pragma) mode diff --git a/tests/cbor/testObjects.nim b/tests/cbor/testObjects.nim index 2f8415e..d9f9f17 100644 --- a/tests/cbor/testObjects.nim +++ b/tests/cbor/testObjects.nim @@ -24,7 +24,7 @@ type s: string nums: seq[int] - CompositeNested = object + CompositeNested {.deserialize.} = object u: uint64 n: int b: seq[byte] @@ -37,15 +37,16 @@ type coordinates: tuple[x: int, y: int, label: string] refInner: ref Inner -proc fromCborHook*(v: var CustomColor, n: CborNode): ?!void = +proc fromCbor*(_: type CustomColor, n: CborNode): ?!CustomColor = + var v: CustomColor if n.kind == cborNegative: v = CustomColor(n.int) - result = success() + success(v) else: - result = failure(newSerdeError("Expected signed integer, got " & $n.kind)) + failure(newSerdeError("Expected signed integer, got " & $n.kind)) # Custom fromCborHook for CustomPoint -proc fromCborHook*(v: var CustomPoint, n: CborNode): ?!void = +proc fromCbor*(_: type CustomPoint, n: CborNode): ?!CustomPoint = if n.kind == cborArray and n.seq.len == 2: var x, y: int let xResult = fromCbor(x, n.seq[0]) @@ -56,10 +57,9 @@ proc fromCborHook*(v: var CustomPoint, n: CborNode): ?!void = if yResult.isFailure: return failure(yResult.error) - v = CustomPoint(x: x, y: y) - result = success() + return success(CustomPoint(x: x, y: y)) else: - result = failure(newSerdeError("Expected array of length 2 for CustomPoint")) + return failure(newSerdeError("Expected array of length 2 for CustomPoint")) # Helper function to create CBOR data for testing proc createPointCbor(x, y: int): CborNode = @@ -97,10 +97,11 @@ suite "CBOR deserialization": let node = createObjectCbor("Test Object", point, Green) # Deserialize - var deserializedObj: CustomObject + # var deserializedObj: CustomObject # Check result - let result = fromCbor(deserializedObj, node) + let result = CustomObject.fromCbor(node) check result.isSuccess + var deserializedObj = result.tryValue check deserializedObj.name == "Test Object" check deserializedObj.point.x == 15 check deserializedObj.point.y == 25 @@ -148,10 +149,9 @@ suite "CBOR deserialization": let node = parseResult.tryValue # Deserialize to CompositeNested object - var roundtrip: CompositeNested - let deserResult = fromCbor(roundtrip, node) - check deserResult.isSuccess - + let res = CompositeNested.fromCbor(node) + check res.isSuccess + let roundtrip = res.tryValue # Check top-level fields check roundtrip.u == original.u check roundtrip.n == original.n diff --git a/tests/cbor/testPrimitives.nim b/tests/cbor/testPrimitives.nim index 2741390..5cf8044 100644 --- a/tests/cbor/testPrimitives.nim +++ b/tests/cbor/testPrimitives.nim @@ -27,7 +27,7 @@ suite "decode": controlCbor = base64.decode v["cbor"].getStr without c =? parseCbor(controlCbor), error: fail() - let js = c.toJsonHook() + let js = c.toJson() if js.isNil: fail() else: From a96971968d62aef16fe75d9b69f13bb5f284b98d Mon Sep 17 00:00:00 2001 From: munna0908 Date: Fri, 30 May 2025 15:24:39 +0530 Subject: [PATCH 10/20] refactor: split writeCbor monolith into specialized type implementations --- serde/cbor/deserializer.nim | 230 ++++++++++++++------------- serde/cbor/serializer.nim | 288 ++++++++++++++++++---------------- tests/cbor/testObjects.nim | 167 +++++++++++++------- tests/cbor/testPrimitives.nim | 3 +- 4 files changed, 385 insertions(+), 303 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 0b3432d..eacf5be 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -2,9 +2,9 @@ # originally available at https://github.com/ehmry/cbor-nim and released under The Unlicense. import std/[math, streams, options, tables, strutils, times, typetraits, macros] -import ./types +import ./types as cborTypes import ./helpers -import ../utils/types as utilsTypes +import ../utils/types import ../utils/pragmas import ./errors import pkg/questionable @@ -13,8 +13,8 @@ import pkg/questionable/results export results export types export pragmas -export utilsTypes - +export cborTypes +export macros {.push raises: [].} @@ -636,117 +636,126 @@ proc fromCbor*[T: tuple](_: type T; n: CborNode): ?!T = success res -proc fromCbor*[T](v: var T; n: CborNode): ?!void = - try: - when T is CborNode: - v = n - result = success() - elif compiles(fromCborHook(v, n)): - return fromCborHook(v, n) - elif T is distinct: - return fromCbor(distinctBase v, n) - elif T is SomeUnsignedInt: - expectCborKind(T, {cborUnsigned}, n) - v = T n.uint - if v.BiggestUInt == n.uint: - return success() - else: - return failure(newCborError("Value overflow for unsigned integer")) - elif T is SomeSignedInt: - expectCborKind(T, {cborUnsigned, cborNegative}, n) - if n.kind == cborUnsigned: - v = T n.uint - if v.BiggestUInt == n.uint: - return success() - else: - return failure(newCborError("Value overflow for un signed integer")) - elif n.kind == cborNegative: - v = T n.int - if v.BiggestInt == n.int: - return success() - else: - return failure(newCborError("Value overflow for signed integer")) - elif T is bool: - if not n.isBool: - return failure(newCborError("Expected boolean, got " & $n.kind)) - v = n.getBool - return success() - elif T is SomeFloat: - expectCborKind(T, {cborFloat}, n) - v = T n.float - return success() - elif T is seq[byte]: - expectCborKind(T, {cborBytes}, n) - v = n.bytes - return success() - elif T is string: - expectCborKind(T, {cborText}, n) - v = n.text - return success() - elif T is seq: - expectCborKind(T, {cborArray}, n) - v.setLen n.seq.len - for i, e in n.seq: - let itemResult = fromCbor(v[i], e) - if itemResult.isFailure: - v.setLen 0 - return failure(itemResult.error) - return success() - elif T is tuple: - expectCborKind(T, {cborArray}, n) - if n.seq.len != T.tupleLen: - return failure(newCborError("Expected tuple of length " & $T.tupleLen)) - var i: int - for f in fields(v): - let itemResult = fromCbor(f, n.seq[i]) - if itemResult.isFailure: - return failure(itemResult.error) - inc i - return success() - elif T is ref: - if n.isNull: - v = nil - return success() - else: - if isNil(v): new(v) - return fromCbor(v[], n) - elif T is object: - expectCborKind(T, {cborMap}, n) - var - i: int - key = CborNode(kind: cborText) - for s, _ in fieldPairs(v): - key.text = s - if not n.map.hasKey key: - return failure(newCborError("Missing field: " & s)) - else: - let fieldResult = fromCbor(v.dot(s), n.map[key]) - if fieldResult.isFailure: - return failure(fieldResult.error) - inc i - if i == n.map.len: - return success() - else: - return failure(newCborError("Extra fields in map")) - else: - return failure(newCborError("Unsupported type: " & $T)) - except CatchableError as e: - return failure newCborError(e.msg) - except Exception as e: - raise newException(Defect, e.msg, e) +# proc fromCbor*[T](v: var T; n: CborNode): ?!void = +# try: +# when T is CborNode: +# v = n +# result = success() +# elif compiles(fromCborHook(v, n)): +# return fromCborHook(v, n) +# elif T is distinct: +# return fromCbor(distinctBase v, n) +# elif T is SomeUnsignedInt: +# expectCborKind(T, {cborUnsigned}, n) +# v = T n.uint +# if v.BiggestUInt == n.uint: +# return success() +# else: +# return failure(newCborError("Value overflow for unsigned integer")) +# elif T is SomeSignedInt: +# expectCborKind(T, {cborUnsigned, cborNegative}, n) +# if n.kind == cborUnsigned: +# v = T n.uint +# if v.BiggestUInt == n.uint: +# return success() +# else: +# return failure(newCborError("Value overflow for un signed integer")) +# elif n.kind == cborNegative: +# v = T n.int +# if v.BiggestInt == n.int: +# return success() +# else: +# return failure(newCborError("Value overflow for signed integer")) +# elif T is bool: +# if not n.isBool: +# return failure(newCborError("Expected boolean, got " & $n.kind)) +# v = n.getBool +# return success() +# elif T is SomeFloat: +# expectCborKind(T, {cborFloat}, n) +# v = T n.float +# return success() +# elif T is seq[byte]: +# expectCborKind(T, {cborBytes}, n) +# v = n.bytes +# return success() +# elif T is string: +# expectCborKind(T, {cborText}, n) +# v = n.text +# return success() +# elif T is seq: +# expectCborKind(T, {cborArray}, n) +# v.setLen n.seq.len +# for i, e in n.seq: +# let itemResult = fromCbor(v[i], e) +# if itemResult.isFailure: +# v.setLen 0 +# return failure(itemResult.error) +# return success() +# elif T is tuple: +# expectCborKind(T, {cborArray}, n) +# if n.seq.len != T.tupleLen: +# return failure(newCborError("Expected tuple of length " & $T.tupleLen)) +# var i: int +# for f in fields(v): +# let itemResult = fromCbor(f, n.seq[i]) +# if itemResult.isFailure: +# return failure(itemResult.error) +# inc i +# return success() +# elif T is ref: +# if n.isNull: +# v = nil +# return success() +# else: +# if isNil(v): new(v) +# return fromCbor(v[], n) +# elif T is object: +# expectCborKind(T, {cborMap}, n) +# var +# i: int +# key = CborNode(kind: cborText) +# for s, _ in fieldPairs(v): +# key.text = s +# if not n.map.hasKey key: +# return failure(newCborError("Missing field: " & s)) +# else: +# let fieldResult = fromCbor(v.dot(s), n.map[key]) +# if fieldResult.isFailure: +# return failure(fieldResult.error) +# inc i +# if i == n.map.len: +# return success() +# else: +# return failure(newCborError("Extra fields in map")) +# else: +# return failure(newCborError("Unsupported type: " & $T)) +# except CatchableError as e: +# return failure newCborError(e.msg) +# except Exception as e: +# raise newException(Defect, e.msg, e) -proc fromCbor*[T: ref object or object](_: type T; n: CborNode): ?!T = +proc fromCbor*[T: ref](_: type T; n: CborNode): ?!T = + when T is ref: + if n.isNull: + return success(T.default) + else: + var resRef = T.new() + let res = typeof(resRef[]).fromCbor(n) + if res.isFailure: + return failure(newCborError(res.error.msg)) + resRef[] = res.value + return success(resRef) + +proc fromCbor*[T: object](_: type T; n: CborNode): ?!T = when T is CborNode: return success T(n) expectCborKind(T, {cborMap}, n) - var res = - when type(T) is ref: - T.new() - else: - T.default + var res = T.default + let mode = getSerdeMode(T, deserialize) try: var i: int @@ -772,3 +781,8 @@ proc fromCbor*[T: ref object or object](_: type T; n: CborNode): ?!T = return failure newCborError(e.msg) except Exception as e: raise newException(Defect, e.msg, e) + +proc fromCbor*[T: ref object or object](_: type T; str: string): ?!T = + var n = ?parseCbor(str) + T.fromCbor(n) + diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index 2c0b014..d557931 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -111,152 +111,168 @@ proc writeCbor*(str: Stream; buf: pointer; len: int): ?!void = ?str.writeInitial(BytesMajor, len) if len > 0: return catch str.writeData(buf, len) + success() proc isSorted*(n: CborNode): ?!bool {.gcsafe.} -proc writeCbor*[T](str: Stream; v: T): ?!void = - ## Write the CBOR binary representation of a `T` to a `Stream`. - ## The behavior of this procedure can be extended or overriden - ## by defining `proc writeCborHook(str: Stream; v: T)` for specific - ## types `T`. +proc writeCbor(str: Stream; v: SomeUnsignedInt): ?!void = + str.writeInitial(0, v) + +proc writeCbor*(str: Stream; v: SomeSignedInt): ?!void = + if v < 0: + ?str.writeInitial(1, -1-v) + else: + ?str.writeInitial(0, v) + success() + +proc writeCbor*(str: Stream; v: seq[byte]): ?!void = + ?str.writeInitial(BytesMajor, v.len) + if v.len > 0: + return catch str.writeData(unsafeAddr v[0], v.len) + success() + +proc writeCbor*(str: Stream; v: string): ?!void = + ?str.writeInitial(TextMajor, v.len) + return catch str.write(v) + +proc writeCbor*[T: char or uint8 or int8](str: Stream; v: openArray[T]): ?!void = + ?str.writeInitial(BytesMajor, v.len) + if v.len > 0: + return catch str.writeData(unsafeAddr v[0], v.len) + success() + +proc writeCbor*[T: array or seq](str: Stream; v: T): ?!void = + ?str.writeInitial(4, v.len) + for e in v.items: + ?str.writeCbor(e) + success() + +proc writeCbor*(str: Stream; v: tuple): ?!void = + ?str.writeInitial(4, v.tupleLen) + for e in v.fields: + ?str.writeCbor(e) + success() + +proc writeCbor*[T: ptr | ref](str: Stream; v: T): ?!void = + if system.`==`(v, nil): + # Major type 7 + return catch str.write(Null) + else: + ?str.writeCbor(v[]) + success() + +proc writeCbor*(str: Stream; v: bool): ?!void = + return catch str.write(initialByte(7, (if v: 21 else: 20))) + +proc writeCbor*[T: SomeFloat](str: Stream; v: T): ?!void = try: - when T is CborNode: - if v.tag.isSome: - ?str.writeCborTag(v.tag.get) - case v.kind: - of cborUnsigned: - return str.writeCbor(v.uint) - of cborNegative: - return str.writeCbor(v.int) - of cborBytes: - ?str.writeInitial(cborBytes.uint8, v.bytes.len) - for b in v.bytes.items: - str.write(b) - of cborText: - ?str.writeInitial(cborText.uint8, v.text.len) - str.write(v.text) - of cborArray: - ?str.writeInitial(4, v.seq.len) - for e in v.seq: - ?str.writeCbor(e) - of cborMap: - without isSortedRes =? v.isSorted, error: - return failure(error) - if not isSortedRes: - return failure(newSerdeError("refusing to write unsorted map to stream")) - ?str.writeInitial(5, v.map.len) - for k, f in v.map.pairs: - ?str.writeCbor(k) - ?str.writeCbor(f) - of cborTag: - discard - of cborSimple: - if v.simple > 31'u or v.simple == 24: - str.write(initialByte(cborSimple.uint8, 24)) - str.write(v.simple) - else: - str.write(initialByte(cborSimple.uint8, v.simple)) - of cborFloat: - return str.writeCbor(v.float) - of cborRaw: - str.write(v.raw) - elif compiles(writeCborHook(str, v)): - ?writeCborHook(str, v) - elif T is SomeUnsignedInt: - ?str.writeInitial(0, v) - elif T is SomeSignedInt: - if v < 0: - # Major type 1 - ?str.writeInitial(1, -1 - v) - else: - # Major type 0 - ?str.writeInitial(0, v) - elif T is seq[byte]: - ?str.writeInitial(BytesMajor, v.len) - if v.len > 0: - str.writeData(unsafeAddr v[0], v.len) - elif T is openArray[char | uint8 | int8]: - ?str.writeInitial(BytesMajor, v.len) - if v.len > 0: - str.writeData(unsafeAddr v[0], v.len) - elif T is string: - ?str.writeInitial(TextMajor, v.len) - str.write(v) - elif T is array | seq: - ?str.writeInitial(4, v.len) - for e in v.items: - ?str.writeCbor(e) - elif T is tuple: - ?str.writeInitial(4, T.tupleLen) - for f in v.fields: ?str.writeCbor(f) - elif T is ptr | ref: - if system.`==`(v, nil): - # Major type 7 - str.write(Null) - else: ?str.writeCbor(v[]) - elif T is object: - var n: uint - for _, _ in v.fieldPairs: - inc n - ?str.writeInitial(5, n) - for k, f in v.fieldPairs: - ?str.writeCbor(k) - ?str.writeCbor(f) - elif T is bool: - str.write(initialByte(7, (if v: 21 else: 20))) - elif T is SomeFloat: - case v.classify - of fcNormal, fcSubnormal: - let single = v.float32 - if single.float64 == v.float64: - if single.isHalfPrecise: - let half = floatHalf(single) - str.write(initialByte(7, 25)) - when system.cpuEndian == bigEndian: - str.write(half) - else: - var be: uint16 - swapEndian16 be.addr, half.unsafeAddr - str.write(be) - else: - str.write initialByte(7, 26) - when system.cpuEndian == bigEndian: - str.write(single) - else: - var be: uint32 - swapEndian32 be.addr, single.unsafeAddr - str.write(be) - else: - str.write initialByte(7, 27) + case v.classify + of fcNormal, fcSubnormal: + let single = v.float32 + if single.float64 == v.float64: + if single.isHalfPrecise: + let half = floatHalf(single) + str.write(initialByte(7, 25)) when system.cpuEndian == bigEndian: - str.write(v) + str.write(half) else: - var be: float64 - swapEndian64 be.addr, v.unsafeAddr - str.write be - return success() - of fcZero: - str.write initialByte(7, 25) - str.write((char)0x00) - of fcNegZero: - str.write initialByte(7, 25) - str.write((char)0x80) - of fcInf: - str.write initialByte(7, 25) - str.write((char)0x7c) - of fcNan: - str.write initialByte(7, 25) - str.write((char)0x7e) - of fcNegInf: - str.write initialByte(7, 25) - str.write((char)0xfc) + var be: uint16 + swapEndian16 be.addr, half.unsafeAddr + str.write(be) + else: + str.write initialByte(7, 26) + when system.cpuEndian == bigEndian: + str.write(single) + else: + var be: uint32 + swapEndian32 be.addr, single.unsafeAddr + str.write(be) + else: + str.write initialByte(7, 27) + when system.cpuEndian == bigEndian: + str.write(v) + else: + var be: uint64 + swapEndian64 be.addr, v.unsafeAddr + str.write(be) + return success() + of fcZero: + str.write initialByte(7, 25) str.write((char)0x00) + of fcNegZero: + str.write initialByte(7, 25) + str.write((char)0x80) + of fcInf: + str.write initialByte(7, 25) + str.write((char)0x7c) + of fcNan: + str.write initialByte(7, 25) + str.write((char)0x7e) + of fcNegInf: + str.write initialByte(7, 25) + str.write((char)0xfc) + str.write((char)0x00) success() except IOError as io: return failure(io.msg) except OSError as os: return failure(os.msg) +proc writeCbor*(str: Stream; v: CborNode): ?!void = + try: + if v.tag.isSome: + ?str.writeCborTag(v.tag.get) + case v.kind: + of cborUnsigned: + ?str.writeCbor(v.uint) + of cborNegative: + ?str.writeCbor(v.int) + of cborBytes: + ?str.writeInitial(cborBytes.uint8, v.bytes.len) + for b in v.bytes.items: + str.write(b) + of cborText: + ?str.writeInitial(cborText.uint8, v.text.len) + str.write(v.text) + of cborArray: + ?str.writeInitial(4, v.seq.len) + for e in v.seq: + ?str.writeCbor(e) + of cborMap: + without isSortedRes =? v.isSorted, error: + return failure(error) + if not isSortedRes: + return failure(newSerdeError("refusing to write unsorted map to stream")) + ?str.writeInitial(5, v.map.len) + for k, f in v.map.pairs: + ?str.writeCbor(k) + ?str.writeCbor(f) + of cborTag: + discard + of cborSimple: + if v.simple > 31'u or v.simple == 24: + str.write(initialByte(cborSimple.uint8, 24)) + str.write(v.simple) + else: + str.write(initialByte(cborSimple.uint8, v.simple)) + of cborFloat: + ?str.writeCbor(v.float) + of cborRaw: + str.write(v.raw) + success() + except CatchableError as e: + return failure(e.msg) + +proc writeCbor*[T: object](str: Stream; v: T): ?!void = + var n: uint + for _, _ in v.fieldPairs: + inc n + ?str.writeInitial(5, n) + for k, f in v.fieldPairs: + ?str.writeCbor(k) + ?str.writeCbor(f) + success() + proc writeCborArray*(str: Stream; args: varargs[CborNode, toCbor]): ?!void = ## Encode to a CBOR array in binary form. This magic doesn't ## always work, some arguments may need to be explicitly @@ -314,14 +330,14 @@ proc sort*(n: var CborNode): ?!void = except Exception as e: raise newException(Defect, e.msg, e) -proc writeCborHook*(str: Stream; dt: DateTime): ?!void = +proc writeCbor*(str: Stream; dt: DateTime): ?!void = ## Write a `DateTime` using the tagged string representation ## defined in RCF7049 section 2.4.1. ?writeCborTag(str, 0) ?writeCbor(str, format(dt, dateTimeFormat)) success() -proc writeCborHook*(str: Stream; t: Time): ?!void = +proc writeCbor*(str: Stream; t: Time): ?!void = ## Write a `Time` using the tagged numerical representation ## defined in RCF7049 section 2.4.1. ?writeCborTag(str, 1) @@ -407,5 +423,3 @@ func initCborMap*(initialSize = tables.defaultInitialSize): CborNode = func initCbor*(items: varargs[CborNode, toCbor]): CborNode = ## Initialize a CBOR arrary. CborNode(kind: cborArray, seq: @items) - - diff --git a/tests/cbor/testObjects.nim b/tests/cbor/testObjects.nim index d9f9f17..ab22498 100644 --- a/tests/cbor/testObjects.nim +++ b/tests/cbor/testObjects.nim @@ -1,42 +1,67 @@ - import std/unittest import std/options import std/streams +import std/macros import pkg/serde import pkg/questionable import pkg/questionable/results -# Custom type for testing +#[ + Test types definitions + These types are used to test various aspects of CBOR serialization/deserialization: + - Basic types (integers, strings, etc.) + - Custom types with custom serialization logic + - Nested objects + - Reference types + - Collections (sequences, tuples) +]# type + # A simple 2D point with x and y coordinates CustomPoint = object x: int y: int + # Enum type to test enum serialization CustomColor = enum Red, Green, Blue + # Object combining different custom types CustomObject = object name: string point: CustomPoint color: CustomColor + # Simple object with a string and sequence Inner = object s: string nums: seq[int] - CompositeNested {.deserialize.} = object - u: uint64 - n: int - b: seq[byte] - t: string - arr: seq[int] - tag: float - flag: bool - inner: Inner - innerArr: seq[Inner] - coordinates: tuple[x: int, y: int, label: string] - refInner: ref Inner + # Reference type for testing ref object serialization + NewType = ref object + size: uint64 + # Reference type for Inner object + InnerRef = ref Inner + + # Complex object with various field types to test comprehensive serialization + CompositeNested = object + u: uint64 # Unsigned integer + n: int # Signed integer + b: seq[byte] # Byte sequence + t: string # Text string + arr: seq[int] # Integer sequence + tag: float # Floating point + flag: bool # Boolean + inner: Inner # Nested object + innerArr: seq[Inner] # Sequence of objects + coordinates: tuple[x: int, y: int, label: string] # Tuple + refInner: ref Inner # Reference to object + refNewInner: NewType # Custom reference type + refNil: ref Inner # Nil reference + customPoint: CustomPoint # Custom type + +# Custom deserialization for CustomColor enum +# Converts a CBOR negative integer to a CustomColor enum value proc fromCbor*(_: type CustomColor, n: CborNode): ?!CustomColor = var v: CustomColor if n.kind == cborNegative: @@ -45,22 +70,28 @@ proc fromCbor*(_: type CustomColor, n: CborNode): ?!CustomColor = else: failure(newSerdeError("Expected signed integer, got " & $n.kind)) -# Custom fromCborHook for CustomPoint +# Custom deserialization for CustomPoint +# Expects a CBOR array with exactly 2 elements representing x and y coordinates proc fromCbor*(_: type CustomPoint, n: CborNode): ?!CustomPoint = if n.kind == cborArray and n.seq.len == 2: - var x, y: int - let xResult = fromCbor(x, n.seq[0]) - if xResult.isFailure: - return failure(xResult.error) - - let yResult = fromCbor(y, n.seq[1]) - if yResult.isFailure: - return failure(yResult.error) + let x = ?int.fromCbor(n.seq[0]) + let y = ?int.fromCbor(n.seq[1]) return success(CustomPoint(x: x, y: y)) else: return failure(newSerdeError("Expected array of length 2 for CustomPoint")) +# Custom serialization for CustomPoint +# Serializes a CustomPoint as a CBOR array with 2 elements: [x, y] +proc writeCbor*(str: Stream, val: CustomPoint): ?!void = + # Write array header with length 2 + ?str.writeCborArrayLen(2) + + # Write x and y coordinates + ?str.writeCbor(val.x) + + str.writeCbor(val.y) + # Helper function to create CBOR data for testing proc createPointCbor(x, y: int): CborNode = result = CborNode(kind: cborArray) @@ -69,6 +100,7 @@ proc createPointCbor(x, y: int): CborNode = CborNode(kind: cborUnsigned, uint: y.uint64) ] +# Creates a CBOR map node representing a CustomObject proc createObjectCbor(name: string, point: CustomPoint, color: CustomColor): CborNode = result = CborNode(kind: cborMap) @@ -89,19 +121,20 @@ proc createObjectCbor(name: string, point: CustomPoint, suite "CBOR deserialization": test "deserializes object with custom types": - # Create a complex object + # Create a test point let point = CustomPoint(x: 15, y: 25) - # let obj = CustomObject(name: "Test Object", point: point, color: Green) - # Create CBOR representation + # Create CBOR representation of a CustomObject let node = createObjectCbor("Test Object", point, Green) - # Deserialize - # var deserializedObj: CustomObject - # Check result + # Deserialize CBOR to CustomObject let result = CustomObject.fromCbor(node) + + # Verify deserialization was successful check result.isSuccess var deserializedObj = result.tryValue + + # Verify all fields were correctly deserialized check deserializedObj.name == "Test Object" check deserializedObj.point.x == 15 check deserializedObj.point.y == 25 @@ -109,73 +142,95 @@ suite "CBOR deserialization": test "serialize and deserialize object with all supported wire types": - var refObj = new Inner - refObj.s = "refInner" - refObj.nums = @[30, 40] + # Setup test data with various types + # 1. Create reference objects + var refInner = new Inner + refInner.s = "refInner" + refInner.nums = @[30, 40] + + var refNewObj = new NewType + refNewObj.size = 42 + + # 2. Create a complex object with all supported types var original = CompositeNested( - u: 42, - n: -99, - b: @[byte 1, byte 2], - t: "hi", - arr: @[1, 2, 3], - tag: 1.5, - flag: true, - inner: Inner(s: "inner!", nums: @[10, 20]), - innerArr: @[ + u: 42, # unsigned integer + n: -99, # signed integer + b: @[byte 1, byte 2], # byte array + t: "hi", # string + arr: @[1, 2, 3], # integer array + tag: 1.5, # float + flag: true, # boolean + inner: Inner(s: "inner!", nums: @[10, 20]), # nested object + innerArr: @[ # array of objects Inner(s: "first", nums: @[1, 2]), Inner(s: "second", nums: @[3, 4, 5]) ], - coordinates: (x: 10, y: 20, label: "test"), - refInner: refObj + coordinates: (x: 10, y: 20, label: "test"), # tuple + refInner: refInner, # reference to object + refNewInner: refNewObj, # custom reference type + refNil: nil, # nil reference + customPoint: CustomPoint(x: 15, y: 25) # custom type ) - # Serialize to CBOR with encode + # Test serialization using encode helper without encodedStr =? encode(original), error: fail() - # Serialize to CBOR + # Test serialization using stream API let stream = newStringStream() - if stream.writeCbor(original).isFailure: - fail() + check not stream.writeCbor(original).isFailure + # Get the serialized CBOR data let cborData = stream.data - # Check that both serialized forms are equal + + # Verify both serialization methods produce the same result check cborData == encodedStr - # Parse CBOR back to CborNode + # Parse CBOR data back to CborNode let parseResult = parseCbor(cborData) - check parseResult.isSuccess let node = parseResult.tryValue - # Deserialize to CompositeNested object + # Deserialize CborNode to CompositeNested object let res = CompositeNested.fromCbor(node) check res.isSuccess let roundtrip = res.tryValue - # Check top-level fields + + # Verify all fields were correctly round-tripped + + # 1. Check primitive fields check roundtrip.u == original.u check roundtrip.n == original.n check roundtrip.b == original.b check roundtrip.t == original.t check roundtrip.arr == original.arr - check abs(roundtrip.tag - original.tag) < 1e-6 + check abs(roundtrip.tag - original.tag) < 1e-6 # Float comparison with epsilon check roundtrip.flag == original.flag - # Check nested object + # 2. Check nested object fields check roundtrip.inner.s == original.inner.s check roundtrip.inner.nums == original.inner.nums - # Check nested array of objects + # 3. Check sequence of objects check roundtrip.innerArr.len == original.innerArr.len for i in 0.. Date: Fri, 30 May 2025 15:53:57 +0530 Subject: [PATCH 11/20] add tests for Time and TimeDate --- serde/cbor/deserializer.nim | 14 ++++++++------ tests/cbor/testObjects.nim | 7 ++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index eacf5be..8721c03 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -494,30 +494,32 @@ proc parseTime(n: CborNode): Time = else: assert false -proc fromCborHook*(v: var DateTime; n: CborNode): ?!void = +proc fromCbor*(_: type DateTime; n: CborNode): ?!DateTime = ## Parse a `DateTime` from the tagged string representation ## defined in RCF7049 section 2.4.1. + var v: DateTime if n.tag.isSome: try: if n.tag.get == 0 and n.kind == cborText: v = parseDateText(n) - return success() + return success(v) elif n.tag.get == 1 and n.kind in {cborUnsigned, cborNegative, cborFloat}: v = parseTime(n).utc - return success() + return success(v) except ValueError as e: return failure(e) -proc fromCborHook*(v: var Time; n: CborNode): ?!void = +proc fromCbor*(_: type Time; n: CborNode): ?!Time = ## Parse a `Time` from the tagged string representation ## defined in RCF7049 section 2.4.1. + var v: Time if n.tag.isSome: try: if n.tag.get == 0 and n.kind == cborText: v = parseDateText(n).toTime - return success() + return success(v) elif n.tag.get == 1 and n.kind in {cborUnsigned, cborNegative, cborFloat}: v = parseTime(n) - return success() + return success(v) except ValueError as e: return failure(e) func isTagged*(n: CborNode): bool = diff --git a/tests/cbor/testObjects.nim b/tests/cbor/testObjects.nim index ab22498..04485fa 100644 --- a/tests/cbor/testObjects.nim +++ b/tests/cbor/testObjects.nim @@ -1,6 +1,7 @@ import std/unittest import std/options import std/streams +import std/times import std/macros import pkg/serde import pkg/questionable @@ -59,6 +60,8 @@ type refNewInner: NewType # Custom reference type refNil: ref Inner # Nil reference customPoint: CustomPoint # Custom type + time: Time # Time + date: DateTime # DateTime # Custom deserialization for CustomColor enum # Converts a CBOR negative integer to a CustomColor enum value @@ -169,7 +172,9 @@ suite "CBOR deserialization": refInner: refInner, # reference to object refNewInner: refNewObj, # custom reference type refNil: nil, # nil reference - customPoint: CustomPoint(x: 15, y: 25) # custom type + customPoint: CustomPoint(x: 15, y: 25), # custom type + time: getTime(), # time + date: now().utc # date ) # Test serialization using encode helper From 9d3da40c0a6682b100c1d5c3d32f490b3b7642cc Mon Sep 17 00:00:00 2001 From: munna0908 Date: Sat, 31 May 2025 17:38:35 +0530 Subject: [PATCH 12/20] feat: refactor CBOR parser to use direct exceptions instead of Result type --- config.nims | 6 +- serde/cbor/deserializer.nim | 438 +++++++++++++---------------- serde/cbor/errors.nim | 7 +- serde/cbor/helpers.nim | 33 ++- serde/cbor/jsonhook.nim | 72 ++--- serde/cbor/serializer.nim | 87 +++--- serde/cbor/types.nim | 55 ++-- serde/json/deserializer.nim | 18 +- serde/json/errors.nim | 3 +- serde/json/helpers.nim | 6 +- serde/json/stdjson.nim | 2 +- serde/utils/errors.nim | 3 - serde/utils/pragmas.nim | 15 +- tests/benchmark.nim | 74 +++++ tests/cbor/testObjects.nim | 93 +++--- tests/cbor/testPrimitives.nim | 44 ++- tests/config.nims | 2 +- tests/json/deserialize/objects.nim | 3 +- tests/json/testSerialize.nim | 9 +- tests/test.nimble | 1 - 20 files changed, 498 insertions(+), 473 deletions(-) create mode 100644 tests/benchmark.nim diff --git a/config.nims b/config.nims index e01d099..05a7794 100644 --- a/config.nims +++ b/config.nims @@ -1,5 +1,7 @@ ---styleCheck:usages ---styleCheck:error +--styleCheck: + usages +--styleCheck: + error # begin Nimble config (version 1) when fileExists("nimble.paths"): diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 8721c03..6007104 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -18,10 +18,11 @@ export macros {.push raises: [].} -func isIndefinite*(c: CborParser): bool {.inline.} = c.minor == 31 +func isIndefinite*(c: CborParser): bool {.inline.} = ## Return true if the parser is positioned on an item of indefinite length. + c.minor == 31 -proc open*(c: var CborParser; s: Stream) = +proc open*(c: var CborParser, s: Stream) = ## Begin parsing a stream of CBOR in binary form. ## The parser will be initialized in an EOF state, call ## ``next`` to advance it before parsing. @@ -29,7 +30,7 @@ proc open*(c: var CborParser; s: Stream) = c.kind = cborEof c.intVal = 0 -proc next*(c: var CborParser): ?!void = +proc next*(c: var CborParser) {.raises: [CborParseError].} = ## Advance the parser to the initial or next event. try: if c.s.atEnd: @@ -41,7 +42,7 @@ proc next*(c: var CborParser): ?!void = mb = ib shr 5 c.minor = ib and 0b11111 case c.minor - of 0..23: + of 0 .. 23: c.intVal = c.minor.uint64 of 24: c.intVal = c.s.readChar.uint64 @@ -50,12 +51,12 @@ proc next*(c: var CborParser): ?!void = c.intVal = (c.intVal shl 8) or c.s.readChar.uint64 of 26: c.intVal = c.s.readChar.uint64 - for _ in 1..3: + for _ in 1 .. 3: {.unroll.} c.intVal = (c.intVal shl 8) or c.s.readChar.uint64 of 27: c.intVal = c.s.readChar.uint64 - for _ in 1..7: + for _ in 1 .. 7: {.unroll.} c.intVal = (c.intVal shl 8) or c.s.readChar.uint64 else: @@ -83,333 +84,284 @@ proc next*(c: var CborParser): ?!void = else: c.kind = CborEventKind.cborSimple else: - return failure(newCborError("unhandled major type " & $mb)) - success() + raise newCborError("unhandled major type " & $mb) except IOError as e: - return failure(e) + raise newException(CborParseError, e.msg, e) except OSError as e: - return failure(e) + raise newException(CborParseError, e.msg, e) -proc nextUInt*(c: var CborParser): ?!BiggestUInt = +proc nextUInt*(c: var CborParser): BiggestUInt {.raises: [CborParseError].} = ## Parse the integer value that the parser is positioned on. - if c.kind != CborEventKind.cborPositive: - return failure(newCborError("Expected positive integer, got " & $c.kind)) - let val = c.intVal.BiggestUInt + parseAssert( + c.kind == CborEventKind.cborPositive, "Expected positive integer, got " & $c.kind + ) + result = c.intVal.BiggestUInt - ?c.next() - return success(val) + c.next() -proc nextInt*(c: var CborParser): ?!BiggestInt = +proc nextInt*(c: var CborParser): BiggestInt {.raises: [CborParseError].} = ## Parse the integer value that the parser is positioned on. - var val: BiggestInt case c.kind of CborEventKind.cborPositive: - val = c.intVal.BiggestInt + result = c.intVal.BiggestInt of CborEventKind.cborNegative: - val = -1.BiggestInt - c.intVal.BiggestInt + result = -1.BiggestInt - c.intVal.BiggestInt else: - return failure(newCborError("Expected integer, got " & $c.kind)) + raise newCborError("Expected integer, got " & $c.kind) - ?c.next() - return success(val) + c.next() -proc nextFloat*(c: var CborParser): ?!float64 = +proc nextFloat*(c: var CborParser): float64 {.raises: [CborParseError].} = ## Parse the float value that the parser is positioned on. - var val: float64 - if c.kind != CborEventKind.cborFloat: - return failure(newCborError("Expected float, got " & $c.kind)) + parseAssert(c.kind == CborEventKind.cborFloat, "Expected float, got " & $c.kind) case c.minor of 25: - val = floatSingle(c.intVal.uint16).float64 + result = floatSingle(c.intVal.uint16).float64 of 26: - val = cast[float32](c.intVal).float64 + result = cast[float32](c.intVal).float64 of 27: - val = cast[float64](c.intVal) + result = cast[float64](c.intVal) else: discard - ?c.next() - return success(val) + c.next() -func bytesLen*(c: CborParser): ?!int = +func bytesLen*(c: CborParser): int {.raises: [CborParseError].} = ## Return the length of the byte string that the parser is positioned on. - if c.kind != CborEventKind.cborBytes: - return failure(newCborError("Expected bytes, got " & $c.kind)) - return success(c.intVal.int) + parseAssert(c.kind == CborEventKind.cborBytes, "Expected bytes, got " & $c.kind) + c.intVal.int -proc nextBytes*(c: var CborParser; buf: var openArray[byte]): ?!void = +proc nextBytes*( + c: var CborParser, buf: var openArray[byte] +) {.raises: [CborParseError].} = ## Read the bytes that the parser is positioned on and advance. try: - if c.kind != CborEventKind.cborBytes: - return failure(newCborError("Expected bytes, got " & $c.kind)) - if buf.len != c.intVal.int: - return failure(newCborError("Buffer length mismatch: expected " & - $c.intVal.int & ", got " & $buf.len)) + parseAssert(c.kind == CborEventKind.cborBytes, "Expected bytes, got " & $c.kind) + parseAssert( + buf.len == c.intVal.int, + "Buffer length mismatch: expected " & $c.intVal.int & ", got " & $buf.len, + ) if buf.len > 0: let n = c.s.readData(buf[0].addr, buf.len) - if n != buf.len: - return failure(newCborError("truncated read of CBOR data")) - ?c.next() - success() + parseAssert(n == buf.len, "truncated read of CBOR data") + c.next() except OSError as e: - return failure(e.msg) + raise newException(CborParseError, e.msg, e) except IOError as e: - return failure(e.msg) + raise newException(CborParseError, e.msg, e) -proc nextBytes*(c: var CborParser): ?!seq[byte] = +proc nextBytes*(c: var CborParser): seq[byte] {.raises: [CborParseError].} = ## Read the bytes that the parser is positioned on into a seq and advance. - var val = newSeq[byte](c.intVal.int) - let nextRes = nextBytes(c, val) - if nextRes.isFailure: - return failure(nextRes.error) + result = newSeq[byte](c.intVal.int) + nextBytes(c, result) - return success(val) - -func textLen*(c: CborParser): ?!int = +func textLen*(c: CborParser): int {.raises: [CborParseError].} = ## Return the length of the text that the parser is positioned on. - if c.kind != CborEventKind.cborText: - return failure(newCborError("Expected text, got " & $c.kind)) - return success(c.intVal.int) + parseAssert(c.kind == CborEventKind.cborText, "Expected text, got " & $c.kind) + c.intVal.int -proc nextText*(c: var CborParser; buf: var string): ?!void = +proc nextText*(c: var CborParser, buf: var string) {.raises: [ + CborParseError].} = ## Read the text that the parser is positioned on into a string and advance. try: - if c.kind != CborEventKind.cborText: - return failure(newCborError("Expected text, got " & $c.kind)) + parseAssert(c.kind == CborEventKind.cborText, "Expected text, got " & $c.kind) buf.setLen c.intVal.int if buf.len > 0: let n = c.s.readData(buf[0].addr, buf.len) - if n != buf.len: - return failure(newCborError("truncated read of CBOR data")) - ?c.next() - success() + parseAssert(n == buf.len, "truncated read of CBOR data") + c.next() except IOError as e: - return failure(e.msg) + raise newException(CborParseError, e.msg, e) except OSError as e: - return failure(e.msg) + raise newException(CborParseError, e.msg, e) -proc nextText*(c: var CborParser): ?!string = +proc nextText*(c: var CborParser): string {.raises: [CborParseError].} = ## Read the text that the parser is positioned on into a string and advance. - var buf: string - let nextRes = nextText(c, buf) - if nextRes.isFailure: - return failure(nextRes.error) + nextText(c, result) - return success(buf) - -func arrayLen*(c: CborParser): ?!int = +func arrayLen*(c: CborParser): int {.raises: [CborParseError].} = ## Return the length of the array that the parser is positioned on. - if c.kind != CborEventKind.cborArray: - return failure(newCborError("Expected array, got " & $c.kind)) + parseAssert(c.kind == CborEventKind.cborArray, "Expected array, got " & $c.kind) + c.intVal.int - return success(c.intVal.int) - -func mapLen*(c: CborParser): ?!int = +func mapLen*(c: CborParser): int {.raises: [CborParseError].} = ## Return the length of the map that the parser is positioned on. - if c.kind != CborEventKind.cborMap: - return failure(newCborError("Expected map, got " & $c.kind)) + parseAssert(c.kind == CborEventKind.cborMap, "Expected map, got " & $c.kind) + c.intVal.int - return success(c.intVal.int) - -func tag*(c: CborParser): ?!uint64 = +func tag*(c: CborParser): uint64 {.raises: [CborParseError].} = ## Return the tag value the parser is positioned on. - if c.kind != CborEventKind.cborTag: - return failure(newCborError("Expected tag, got " & $c.kind)) + parseAssert(c.kind == CborEventKind.cborTag, "Expected tag, got " & $c.kind) + c.intVal - return success(c.intVal) - -proc skipNode*(c: var CborParser): ?!void = +proc skipNode*(c: var CborParser) {.raises: [CborParseError].} = ## Skip the item the parser is positioned on. try: case c.kind of CborEventKind.cborEof: - return failure(newCborError("end of CBOR stream")) + raise newCborError("end of CBOR stream") of CborEventKind.cborPositive, CborEventKind.cborNegative, CborEventKind.cborSimple: - return c.next() + c.next() of CborEventKind.cborBytes, CborEventKind.cborText: if c.isIndefinite: - ?c.next() + c.next() while c.kind != CborEventKind.cborBreak: - if c.kind != CborEventKind.cborBytes: - return failure(newCborError("expected bytes, got " & $c.kind)) - for _ in 1..c.intVal.int: discard readChar(c.s) - return c.next() + parseAssert( + c.kind == CborEventKind.cborBytes, "expected bytes, got " & $c.kind + ) + for _ in 1 .. c.intVal.int: + discard readChar(c.s) + c.next() else: - for _ in 1..c.intVal.int: discard readChar(c.s) - return c.next() + for _ in 1 .. c.intVal.int: + discard readChar(c.s) + c.next() of CborEventKind.cborArray: if c.isIndefinite: - ?c.next() + c.next() while c.kind != CborEventKind.cborBreak: - ?c.skipNode() - return c.next() + c.skipNode() + c.next() else: let len = c.intVal - ?c.next() - for i in 1..len: - ?c.skipNode() + c.next() + for i in 1 .. len: + c.skipNode() of CborEventKind.cborMap: let mapLen = c.intVal.int if c.isIndefinite: - ?c.next() + c.next() while c.kind != CborEventKind.cborBreak: - ?c.skipNode() - return c.next() + c.skipNode() + c.next() else: - ?c.next() + c.next() for _ in 1 .. mapLen: - ?c.skipNode() + c.skipNode() of CborEventKind.cborTag: - ?c.next() - return c.skipNode() + c.next() + c.skipNode() of CborEventKind.cborFloat: - without f =? c.nextFloat(), error: - return failure(error) + discard c.nextFloat() of CborEventKind.cborBreak: discard - success() - except OSError as e: - return failure(e.msg) - except IOError as e: - return failure(e.msg) + except OSError as os: + raise newException(CborParseError, os.msg, os) + except IOError as io: + raise newException(CborParseError, io.msg, io) - - -proc nextNode*(c: var CborParser): ?!CborNode = +proc nextNode*(c: var CborParser): CborNode {.raises: [CborParseError].} = ## Parse the item the parser is positioned on into a ``CborNode``. ## This is cheap for numbers or simple values but expensive ## for nested types. try: - var next: CborNode case c.kind of CborEventKind.cborEof: - return failure(newCborError("end of CBOR stream")) + raise newCborError("end of CBOR stream") of CborEventKind.cborPositive: - next = CborNode(kind: cborUnsigned, uint: c.intVal) - ?c.next() + result = CborNode(kind: cborUnsigned, uint: c.intVal) + c.next() of CborEventKind.cborNegative: - next = CborNode(kind: cborNegative, int: -1 - c.intVal.int64) - ?c.next() + result = CborNode(kind: cborNegative, int: -1 - c.intVal.int64) + c.next() of CborEventKind.cborBytes: if c.isIndefinite: - next = CborNode(kind: cborBytes, bytes: newSeq[byte]()) - ?c.next() + result = CborNode(kind: cborBytes, bytes: newSeq[byte]()) + c.next() while c.kind != CborEventKind.cborBreak: - if c.kind != CborEventKind.cborBytes: - return failure(newCborError("Expected bytes, got " & $c.kind)) + parseAssert( + c.kind == CborEventKind.cborBytes, "Expected bytes, got " & $c.kind + ) let chunkLen = c.intVal.int - pos = next.bytes.len - next.bytes.setLen(pos+chunkLen) - let n = c.s.readData(next.bytes[pos].addr, chunkLen) - if n != chunkLen: - return failure(newCborError("truncated read of CBOR data")) - ?c.next() + pos = result.bytes.len + result.bytes.setLen(pos + chunkLen) + let n = c.s.readData(result.bytes[pos].addr, chunkLen) + parseAssert(n == chunkLen, "truncated read of CBOR data") + c.next() else: - without rawBytes =? c.nextBytes(), error: - return failure(error) - next = CborNode(kind: cborBytes, bytes: rawBytes) + result = CborNode(kind: cborBytes, bytes: c.nextBytes()) of CborEventKind.cborText: if c.isIndefinite: - next = CborNode(kind: cborText, text: "") - ?c.next() + result = CborNode(kind: cborText, text: "") + c.next() while c.kind != CborEventKind.cborBreak: - if c.kind != CborEventKind.cborText: - return failure(newCborError("Expected text, got " & $c.kind)) + parseAssert(c.kind == CborEventKind.cborText, "Expected text, got " & $c.kind) let chunkLen = c.intVal.int - pos = next.text.len - next.text.setLen(pos+chunkLen) - let n = c.s.readData(next.text[pos].addr, chunkLen) - if n != chunkLen: - return failure(newCborError("truncated read of CBOR data")) - ?c.next() - ?c.next() + pos = result.text.len + result.text.setLen(pos + chunkLen) + let n = c.s.readData(result.text[pos].addr, chunkLen) + parseAssert(n == chunkLen, "truncated read of CBOR data") + c.next() + c.next() else: - without text =? c.nextText(), error: - return failure(error) - next = CborNode(kind: cborText, text: text) + result = CborNode(kind: cborText, text: c.nextText()) of CborEventKind.cborArray: - next = CborNode(kind: cborArray, seq: newSeq[CborNode](c.intVal)) + result = CborNode(kind: cborArray, seq: newSeq[CborNode](c.intVal)) if c.isIndefinite: - ?c.next() + c.next() while c.kind != CborEventKind.cborBreak: - without node =? c.nextNode(), error: - return failure(error) - next.seq.add(node) - ?c.next() + result.seq.add(c.nextNode()) + c.next() else: - ?c.next() - for i in 0..next.seq.high: - without node =? c.nextNode(), error: - return failure(error) - next.seq[i] = node + c.next() + for i in 0 .. result.seq.high: + result.seq[i] = c.nextNode() of CborEventKind.cborMap: let mapLen = c.intVal.int - next = CborNode(kind: cborMap, map: initOrderedTable[CborNode, CborNode]( - mapLen.nextPowerOfTwo)) + result = CborNode( + kind: cborMap, map: initOrderedTable[CborNode, CborNode]( + mapLen.nextPowerOfTwo) + ) if c.isIndefinite: - ?c.next() + c.next() while c.kind != CborEventKind.cborBreak: - without key =? c.nextNode(), error: - return failure(error) - without val =? c.nextNode(), error: - return failure(error) - next.map[key] = val - ?c.next() + result.map[c.nextNode()] = c.nextNode() + c.next() else: - ?c.next() + c.next() for _ in 1 .. mapLen: - without key =? c.nextNode(), error: - return failure(error) - without val =? c.nextNode(), error: - return failure(error) - next.map[key] = val + result.map[c.nextNode()] = c.nextNode() of CborEventKind.cborTag: let tag = c.intVal - ?c.next() - without node =? c.nextNode(), error: - return failure(error) - next = node - next.tag = some tag + c.next() + result = c.nextNode() + result.tag = some tag of CborEventKind.cborSimple: case c.minor of 24: - next = CborNode(kind: cborSimple, simple: c.intVal.uint8) + result = CborNode(kind: cborSimple, simple: c.intVal.uint8) else: - next = CborNode(kind: cborSimple, simple: c.minor) - ?c.next() + result = CborNode(kind: cborSimple, simple: c.minor) + c.next() of CborEventKind.cborFloat: - without f =? c.nextFloat(), error: - return failure(error) - next = CborNode(kind: cborFloat, float: f) + result = CborNode(kind: cborFloat, float: c.nextFloat()) of CborEventKind.cborBreak: discard - success(next) except OSError as e: - return failure(e.msg) + raise newException(CborParseError, e.msg, e) except IOError as e: - return failure(e.msg) + raise newException(CborParseError, e.msg, e) except CatchableError as e: - return failure(e.msg) + raise newException(CborParseError, e.msg, e) except Exception as e: raise newException(Defect, e.msg, e) - -proc readCbor*(s: Stream): ?!CborNode = +proc readCbor*(s: Stream): CborNode {.raises: [CborParseError].} = ## Parse a stream into a CBOR object. var parser: CborParser parser.open(s) - ?parser.next() + parser.next() parser.nextNode() -proc parseCbor*(s: string): ?!CborNode = +proc parseCbor*(s: string): CborNode {.raises: [CborParseError].} = ## Parse a string into a CBOR object. ## A wrapper over stream parsing. readCbor(newStringStream s) -proc `$`*(n: CborNode): string = +proc `$`*(n: CborNode): string {.raises: [CborParseError].} = ## Get a ``CborNode`` in diagnostic notation. result = "" if n.tag.isSome: @@ -429,7 +381,7 @@ proc `$`*(n: CborNode): string = result.add escape n.text of cborArray: result.add "[" - for i in 0.. 0: @@ -451,12 +403,19 @@ proc `$`*(n: CborNode): string = discard of cborSimple: case n.simple - of 20: result.add "false" - of 21: result.add "true" - of 22: result.add "null" - of 23: result.add "undefined" - of 31: discard # break code for indefinite-length items - else: result.add "simple(" & $n.simple & ")" + of 20: + result.add "false" + of 21: + result.add "true" + of 22: + result.add "null" + of 23: + result.add "undefined" + of 31: + discard + # break code for indefinite-length items + else: + result.add "simple(" & $n.simple & ")" of cborFloat: case n.float.classify of fcNan: @@ -468,14 +427,11 @@ proc `$`*(n: CborNode): string = else: result.add $n.float of cborRaw: - without val =? parseCbor(n.raw), error: - return error.msg - result.add $val + result.add $parseCbor(n.raw) if n.tag.isSome: result.add(")") - -proc getInt*(n: CborNode; default: int = 0): int = +proc getInt*(n: CborNode, default: int = 0): int = ## Get the numerical value of a ``CborNode`` or a fallback. case n.kind of cborUnsigned: n.uint.int @@ -494,7 +450,7 @@ proc parseTime(n: CborNode): Time = else: assert false -proc fromCbor*(_: type DateTime; n: CborNode): ?!DateTime = +proc fromCbor*(_: type DateTime, n: CborNode): ?!DateTime = ## Parse a `DateTime` from the tagged string representation ## defined in RCF7049 section 2.4.1. var v: DateTime @@ -506,9 +462,10 @@ proc fromCbor*(_: type DateTime; n: CborNode): ?!DateTime = elif n.tag.get == 1 and n.kind in {cborUnsigned, cborNegative, cborFloat}: v = parseTime(n).utc return success(v) - except ValueError as e: return failure(e) + except ValueError as e: + return failure(e) -proc fromCbor*(_: type Time; n: CborNode): ?!Time = +proc fromCbor*(_: type Time, n: CborNode): ?!Time = ## Parse a `Time` from the tagged string representation ## defined in RCF7049 section 2.4.1. var v: Time @@ -520,17 +477,18 @@ proc fromCbor*(_: type Time; n: CborNode): ?!Time = elif n.tag.get == 1 and n.kind in {cborUnsigned, cborNegative, cborFloat}: v = parseTime(n) return success(v) - except ValueError as e: return failure(e) + except ValueError as e: + return failure(e) func isTagged*(n: CborNode): bool = ## Check if a CBOR item has a tag. n.tag.isSome -func hasTag*(n: CborNode; tag: Natural): bool = +func hasTag*(n: CborNode, tag: Natural): bool = ## Check if a CBOR item has a tag. n.tag.isSome and n.tag.get == (uint64)tag -proc `tag=`*(result: var CborNode; tag: Natural) = +proc `tag=`*(result: var CborNode, tag: Natural) = ## Tag a CBOR item. result.tag = some(tag.uint64) @@ -541,7 +499,7 @@ func tag*(n: CborNode): uint64 = func isBool*(n: CborNode): bool = (n.kind == cborSimple) and (n.simple in {20, 21}) -func getBool*(n: CborNode; default = false): bool = +func getBool*(n: CborNode, default = false): bool = ## Get the boolean value of a ``CborNode`` or a fallback. if n.kind == cborSimple: case n.simple @@ -555,31 +513,28 @@ func isNull*(n: CborNode): bool = ## Return true if ``n`` is a CBOR null. (n.kind == cborSimple) and (n.simple == 22) -proc getUnsigned*(n: CborNode; default: uint64 = 0): uint64 = +proc getUnsigned*(n: CborNode, default: uint64 = 0): uint64 = ## Get the numerical value of a ``CborNode`` or a fallback. case n.kind of cborUnsigned: n.uint of cborNegative: n.int.uint64 else: default -proc getSigned*(n: CborNode; default: int64 = 0): int64 = +proc getSigned*(n: CborNode, default: int64 = 0): int64 = ## Get the numerical value of a ``CborNode`` or a fallback. case n.kind of cborUnsigned: n.uint.int64 of cborNegative: n.int else: default -func getFloat*(n: CborNode; default = 0.0): float = +func getFloat*(n: CborNode, default = 0.0): float = ## Get the floating-poing value of a ``CborNode`` or a fallback. - if n.kind == cborFloat: - n.float - else: - default + if n.kind == cborFloat: n.float else: default -proc fromCbor*[T: distinct](_: type T; n: CborNode): ?!T = +proc fromCbor*[T: distinct](_: type T, n: CborNode): ?!T = success T(?T.distinctBase.fromCbor(n)) -proc fromCbor*[T: SomeUnsignedInt](_: type T; n: CborNode): ?!T = +proc fromCbor*[T: SomeUnsignedInt](_: type T, n: CborNode): ?!T = expectCborKind(T, {cborUnsigned}, n) var v = T(n.uint) if v.BiggestUInt == n.uint: @@ -587,7 +542,7 @@ proc fromCbor*[T: SomeUnsignedInt](_: type T; n: CborNode): ?!T = else: return failure(newCborError("Value overflow for unsigned integer")) -proc fromCbor*[T: SomeSignedInt](_: type T; n: CborNode): ?!T = +proc fromCbor*[T: SomeSignedInt](_: type T, n: CborNode): ?!T = expectCborKind(T, {cborUnsigned, cborNegative}, n) if n.kind == cborUnsigned: var v = T(n.uint) @@ -602,31 +557,31 @@ proc fromCbor*[T: SomeSignedInt](_: type T; n: CborNode): ?!T = else: return failure(newCborError("Value overflow for signed integer")) -proc fromCbor*[T: SomeFloat](_: type T; n: CborNode): ?!T = +proc fromCbor*[T: SomeFloat](_: type T, n: CborNode): ?!T = expectCborKind(T, {cborFloat}, n) return success(T(n.float)) -proc fromCbor*(_: type seq[byte]; n: CborNode): ?!seq[byte] = +proc fromCbor*(_: type seq[byte], n: CborNode): ?!seq[byte] = expectCborKind(seq[byte], cborBytes, n) return success(n.bytes) -proc fromCbor*(_: type string; n: CborNode): ?!string = +proc fromCbor*(_: type string, n: CborNode): ?!string = expectCborKind(string, cborText, n) return success(n.text) -proc fromCbor*(_: type bool; n: CborNode): ?!bool = +proc fromCbor*(_: type bool, n: CborNode): ?!bool = if not n.isBool: return failure(newCborError("Expected boolean, got " & $n.kind)) return success(n.getBool) -proc fromCbor*[T](_: type seq[T]; n: CborNode): ?!seq[T] = +proc fromCbor*[T](_: type seq[T], n: CborNode): ?!seq[T] = expectCborKind(seq[T], cborArray, n) var arr = newSeq[T](n.seq.len) for i, elem in n.seq: arr[i] = ?T.fromCbor(elem) success arr -proc fromCbor*[T: tuple](_: type T; n: CborNode): ?!T = +proc fromCbor*[T: tuple](_: type T, n: CborNode): ?!T = expectCborKind(T, cborArray, n) var res = T.default if n.seq.len != T.tupleLen: @@ -737,7 +692,7 @@ proc fromCbor*[T: tuple](_: type T; n: CborNode): ?!T = # except Exception as e: # raise newException(Defect, e.msg, e) -proc fromCbor*[T: ref](_: type T; n: CborNode): ?!T = +proc fromCbor*[T: ref](_: type T, n: CborNode): ?!T = when T is ref: if n.isNull: return success(T.default) @@ -749,7 +704,7 @@ proc fromCbor*[T: ref](_: type T; n: CborNode): ?!T = resRef[] = res.value return success(resRef) -proc fromCbor*[T: object](_: type T; n: CborNode): ?!T = +proc fromCbor*[T: object](_: type T, n: CborNode): ?!T = when T is CborNode: return success T(n) @@ -784,7 +739,6 @@ proc fromCbor*[T: object](_: type T; n: CborNode): ?!T = except Exception as e: raise newException(Defect, e.msg, e) -proc fromCbor*[T: ref object or object](_: type T; str: string): ?!T = - var n = ?parseCbor(str) +proc fromCbor*[T: ref object or object](_: type T, str: string): ?!T = + var n = ?(parseCbor(str)).catch T.fromCbor(n) - diff --git a/serde/cbor/errors.nim b/serde/cbor/errors.nim index db6cdc5..e028312 100644 --- a/serde/cbor/errors.nim +++ b/serde/cbor/errors.nim @@ -10,8 +10,7 @@ proc newUnexpectedKindError*( ): ref UnexpectedKindError = newException( UnexpectedKindError, - "deserialization to " & $expectedType & " failed: expected " & - expectedKinds & + "deserialization to " & $expectedType & " failed: expected " & expectedKinds & " but got " & $cbor.kind, ) @@ -37,3 +36,7 @@ proc newUnexpectedKindError*( proc newCborError*(msg: string): ref CborParseError = newException(CborParseError, msg) + +proc parseAssert*(check: bool, msg = "") {.inline.} = + if not check: + raise newException(CborParseError, msg) diff --git a/serde/cbor/helpers.nim b/serde/cbor/helpers.nim index 4332dab..9810c6c 100644 --- a/serde/cbor/helpers.nim +++ b/serde/cbor/helpers.nim @@ -5,22 +5,26 @@ import ./types import ./errors from macros import newDotExpr, newIdentNode, strVal -template expectCborKind*(expectedType: type, expectedKinds: set[CborNodeKind], - cbor: CborNode) = +template expectCborKind*( + expectedType: type, expectedKinds: set[CborNodeKind], cbor: CborNode +) = if cbor.kind notin expectedKinds: return failure(newUnexpectedKindError(expectedType, expectedKinds, cbor)) -template expectCborKind*(expectedType: type, expectedKind: CborNodeKind, - cbor: CborNode) = +template expectCborKind*( + expectedType: type, expectedKind: CborNodeKind, cbor: CborNode +) = expectCborKind(expectedType, {expectedKind}, cbor) -template expectCborKind*(expectedType: type, expectedKinds: set[CborEventKind], - cbor: CborNode) = +template expectCborKind*( + expectedType: type, expectedKinds: set[CborEventKind], cbor: CborNode +) = if cbor.kind notin expectedKinds: return failure(newUnexpectedKindError(expectedType, expectedKinds, cbor)) -template expectCborKind*(expectedType: type, expectedKind: CborEventKind, - cbor: CborNode) = +template expectCborKind*( + expectedType: type, expectedKind: CborEventKind, cbor: CborNode +) = expectCborKind(expectedType, {expectedKind}, cbor) macro dot*(obj: object, fld: string): untyped = @@ -29,15 +33,20 @@ macro dot*(obj: object, fld: string): untyped = func floatSingle*(half: uint16): float32 = ## Convert a 16-bit float to 32-bits. - func ldexp(x: float64; exponent: int): float64 {.importc: "ldexp", - header: "".} + func ldexp( + x: float64, exponent: int + ): float64 {.importc: "ldexp", header: "".} let exp = (half shr 10) and 0x1f mant = float64(half and 0x3ff) - val = if exp == 0: + val = + if exp == 0: ldexp(mant, -24) elif exp != 31: ldexp(mant + 1024, exp.int - 25) else: if mant == 0: Inf else: NaN - if (half and 0x8000) == 0: val else: -val + if (half and 0x8000) == 0: + val + else: + -val diff --git a/serde/cbor/jsonhook.nim b/serde/cbor/jsonhook.nim index faccd1a..fe0b39b 100644 --- a/serde/cbor/jsonhook.nim +++ b/serde/cbor/jsonhook.nim @@ -7,39 +7,39 @@ import ./types import ./errors import ./deserializer -proc toJson*(n: CborNode): JsonNode = - case n.kind: - of cborUnsigned: - newJInt n.uint.BiggestInt - of cborNegative: - newJInt n.int.BiggestInt - of cborBytes: - newJString base64.encode(cast[string](n.bytes), safe = true) - of cborText: - newJString n.text - of cborArray: - let a = newJArray() - for e in n.seq.items: - a.add(e.toJson) - a - of cborMap: - let o = newJObject() - for k, v in n.map.pairs: - if k.kind == cborText: - o[k.text] = v.toJson - else: - o[$k] = v.toJson - o - of cborTag: nil - of cborSimple: - if n.isBool: - newJBool(n.getBool()) - elif n.isNull: - newJNull() - else: nil - of cborFloat: - newJFloat n.float - of cborRaw: - without parsed =? parseCbor(n.raw), error: - raise newCborError(error.msg) - toJson(parsed) +proc toJson*(n: CborNode): JsonNode {.raises: [CborParseError].} = + case n.kind + of cborUnsigned: + newJInt n.uint.BiggestInt + of cborNegative: + newJInt n.int.BiggestInt + of cborBytes: + newJString base64.encode(cast[string](n.bytes), safe = true) + of cborText: + newJString n.text + of cborArray: + let a = newJArray() + for e in n.seq.items: + a.add(e.toJson) + a + of cborMap: + let o = newJObject() + for k, v in n.map.pairs: + if k.kind == cborText: + o[k.text] = v.toJson + else: + o[$k] = v.toJson + o + of cborTag: + nil + of cborSimple: + if n.isBool: + newJBool(n.getBool()) + elif n.isNull: + newJNull() + else: + nil + of cborFloat: + newJFloat n.float + of cborRaw: + toJson(parseCbor(n.raw)) diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index d557931..a222f0d 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -40,7 +40,7 @@ func floatHalf(single: float32): uint16 = func initialByte(major, minor: Natural): uint8 {.inline.} = uint8((major shl 5) or (minor and 0b11111)) -proc writeInitial[T: SomeInteger](str: Stream; m: uint8; n: T): ?!void = +proc writeInitial[T: SomeInteger](str: Stream, m: uint8, n: T): ?!void = ## Write the initial integer of a CBOR item. try: let m = m shl 5 @@ -58,27 +58,27 @@ proc writeInitial[T: SomeInteger](str: Stream; m: uint8; n: T): ?!void = str.write(n.uint8) elif uint64(n) <= uint64(uint16.high): str.write(m or 25'u8) - str.write((uint8)n shr 8) - str.write((uint8)n) + str.write((uint8) n shr 8) + str.write((uint8) n) elif uint64(n) <= uint64(uint32.high): str.write(m or 26'u8) for i in countdown(24, 8, 8): {.unroll.} - str.write((uint8)n shr i) - str.write((uint8)n) + str.write((uint8) n shr i) + str.write((uint8) n) else: str.write(m or 27'u8) for i in countdown(56, 8, 8): {.unroll.} - str.write((uint8)n shr i) - str.write((uint8)n) + str.write((uint8) n shr i) + str.write((uint8) n) success() except IOError as e: return failure(e.msg) except OSError as o: return failure(o.msg) -proc writeCborArrayLen*(str: Stream; len: Natural): ?!void = +proc writeCborArrayLen*(str: Stream, len: Natural): ?!void = ## Write a marker to the stream that initiates an array of ``len`` items. str.writeInitial(4, len) @@ -88,7 +88,7 @@ proc writeCborIndefiniteArrayLen*(str: Stream): ?!void = ## of definite lengths. catch str.write(initialByte(4, 31)) -proc writeCborMapLen*(str: Stream; len: Natural): ?!void = +proc writeCborMapLen*(str: Stream, len: Natural): ?!void = ## Write a marker to the stream that initiates an map of ``len`` pairs. str.writeInitial(5, len) @@ -102,11 +102,11 @@ proc writeCborBreak*(str: Stream): ?!void = ## Write a marker to the stream that ends an indefinite array or map. catch str.write(initialByte(7, 31)) -proc writeCborTag*(str: Stream; tag: Natural): ?!void {.inline.} = +proc writeCborTag*(str: Stream, tag: Natural): ?!void {.inline.} = ## Write a tag for the next CBOR item to a binary stream. str.writeInitial(6, tag) -proc writeCbor*(str: Stream; buf: pointer; len: int): ?!void = +proc writeCbor*(str: Stream, buf: pointer, len: int): ?!void = ## Write a raw buffer to a CBOR `Stream`. ?str.writeInitial(BytesMajor, len) if len > 0: @@ -115,45 +115,45 @@ proc writeCbor*(str: Stream; buf: pointer; len: int): ?!void = proc isSorted*(n: CborNode): ?!bool {.gcsafe.} -proc writeCbor(str: Stream; v: SomeUnsignedInt): ?!void = +proc writeCbor(str: Stream, v: SomeUnsignedInt): ?!void = str.writeInitial(0, v) -proc writeCbor*(str: Stream; v: SomeSignedInt): ?!void = +proc writeCbor*(str: Stream, v: SomeSignedInt): ?!void = if v < 0: - ?str.writeInitial(1, -1-v) + ?str.writeInitial(1, -1 - v) else: ?str.writeInitial(0, v) success() -proc writeCbor*(str: Stream; v: seq[byte]): ?!void = +proc writeCbor*(str: Stream, v: seq[byte]): ?!void = ?str.writeInitial(BytesMajor, v.len) if v.len > 0: return catch str.writeData(unsafeAddr v[0], v.len) success() -proc writeCbor*(str: Stream; v: string): ?!void = +proc writeCbor*(str: Stream, v: string): ?!void = ?str.writeInitial(TextMajor, v.len) return catch str.write(v) -proc writeCbor*[T: char or uint8 or int8](str: Stream; v: openArray[T]): ?!void = +proc writeCbor*[T: char or uint8 or int8](str: Stream, v: openArray[T]): ?!void = ?str.writeInitial(BytesMajor, v.len) if v.len > 0: return catch str.writeData(unsafeAddr v[0], v.len) success() -proc writeCbor*[T: array or seq](str: Stream; v: T): ?!void = +proc writeCbor*[T: array or seq](str: Stream, v: T): ?!void = ?str.writeInitial(4, v.len) for e in v.items: ?str.writeCbor(e) success() -proc writeCbor*(str: Stream; v: tuple): ?!void = +proc writeCbor*(str: Stream, v: tuple): ?!void = ?str.writeInitial(4, v.tupleLen) for e in v.fields: ?str.writeCbor(e) success() -proc writeCbor*[T: ptr | ref](str: Stream; v: T): ?!void = +proc writeCbor*[T: ptr | ref](str: Stream, v: T): ?!void = if system.`==`(v, nil): # Major type 7 return catch str.write(Null) @@ -161,10 +161,10 @@ proc writeCbor*[T: ptr | ref](str: Stream; v: T): ?!void = ?str.writeCbor(v[]) success() -proc writeCbor*(str: Stream; v: bool): ?!void = +proc writeCbor*(str: Stream, v: bool): ?!void = return catch str.write(initialByte(7, (if v: 21 else: 20))) -proc writeCbor*[T: SomeFloat](str: Stream; v: T): ?!void = +proc writeCbor*[T: SomeFloat](str: Stream, v: T): ?!void = try: case v.classify of fcNormal, fcSubnormal: @@ -198,31 +198,31 @@ proc writeCbor*[T: SomeFloat](str: Stream; v: T): ?!void = return success() of fcZero: str.write initialByte(7, 25) - str.write((char)0x00) + str.write((char) 0x00) of fcNegZero: str.write initialByte(7, 25) - str.write((char)0x80) + str.write((char) 0x80) of fcInf: str.write initialByte(7, 25) - str.write((char)0x7c) + str.write((char) 0x7c) of fcNan: str.write initialByte(7, 25) - str.write((char)0x7e) + str.write((char) 0x7e) of fcNegInf: str.write initialByte(7, 25) - str.write((char)0xfc) - str.write((char)0x00) + str.write((char) 0xfc) + str.write((char) 0x00) success() except IOError as io: return failure(io.msg) except OSError as os: return failure(os.msg) -proc writeCbor*(str: Stream; v: CborNode): ?!void = +proc writeCbor*(str: Stream, v: CborNode): ?!void = try: if v.tag.isSome: ?str.writeCborTag(v.tag.get) - case v.kind: + case v.kind of cborUnsigned: ?str.writeCbor(v.uint) of cborNegative: @@ -263,7 +263,7 @@ proc writeCbor*(str: Stream; v: CborNode): ?!void = except CatchableError as e: return failure(e.msg) -proc writeCbor*[T: object](str: Stream; v: T): ?!void = +proc writeCbor*[T: object](str: Stream, v: T): ?!void = var n: uint for _, _ in v.fieldPairs: inc n @@ -273,7 +273,7 @@ proc writeCbor*[T: object](str: Stream; v: T): ?!void = ?str.writeCbor(f) success() -proc writeCborArray*(str: Stream; args: varargs[CborNode, toCbor]): ?!void = +proc writeCborArray*(str: Stream, args: varargs[CborNode, toCbor]): ?!void = ## Encode to a CBOR array in binary form. This magic doesn't ## always work, some arguments may need to be explicitly ## converted with ``toCbor`` before passing. @@ -308,7 +308,8 @@ proc isSorted(n: CborNode): ?!bool = return failure(error.msg) let thisRaw = res.raw if lastRaw != "": - if cmp(lastRaw, thisRaw) > 0: return success(false) + if cmp(lastRaw, thisRaw) > 0: + return success(false) lastRaw = thisRaw success(true) @@ -321,7 +322,7 @@ proc sort*(n: var CborNode): ?!void = return failure(error) if tmp.hasKey(res): tmp[res] = move(val) - sort(tmp) do (x, y: tuple[k: CborNode; v: CborNode]) -> int: + sort(tmp) do(x, y: tuple[k: CborNode, v: CborNode]) -> int: result = cmp(x.k.raw, y.k.raw) n.map = move tmp success() @@ -330,21 +331,22 @@ proc sort*(n: var CborNode): ?!void = except Exception as e: raise newException(Defect, e.msg, e) -proc writeCbor*(str: Stream; dt: DateTime): ?!void = +proc writeCbor*(str: Stream, dt: DateTime): ?!void = ## Write a `DateTime` using the tagged string representation ## defined in RCF7049 section 2.4.1. ?writeCborTag(str, 0) ?writeCbor(str, format(dt, dateTimeFormat)) success() -proc writeCbor*(str: Stream; t: Time): ?!void = +proc writeCbor*(str: Stream, t: Time): ?!void = ## Write a `Time` using the tagged numerical representation ## defined in RCF7049 section 2.4.1. ?writeCborTag(str, 1) ?writeCbor(str, t.toUnix) success() -func toCbor*(x: CborNode): ?!CborNode = success(x) +func toCbor*(x: CborNode): ?!CborNode = + success(x) func toCbor*(x: SomeInteger): ?!CborNode = if x > 0: @@ -369,7 +371,7 @@ func toCbor*(pairs: openArray[(CborNode, CborNode)]): ?!CborNode = except Exception as e: raise newException(Defect, e.msg, e) -func toCbor*(tag: uint64; val: CborNode): ?!CborNode = +func toCbor*(tag: uint64, val: CborNode): ?!CborNode = without res =? toCbor(val), error: return failure(error.msg) var cnode = res @@ -392,11 +394,11 @@ func toCbor*(x: pointer): ?!CborNode = return failure("pointer is not nil") success(CborNode(kind: cborSimple, simple: 22)) -func initCborBytes*[T: char|byte](buf: openArray[T]): CborNode = +func initCborBytes*[T: char | byte](buf: openArray[T]): CborNode = ## Create a CBOR byte string from `buf`. result = CborNode(kind: cborBytes, bytes: newSeq[byte](buf.len)) - for i in 0..= (1, 4): switch("hint", "XCannotRaiseY:off") when (NimMajor, NimMinor, NimPatch) >= (1, 6, 11): - switch("warning", "BareExcept:off") \ No newline at end of file + switch("warning", "BareExcept:off") diff --git a/tests/json/deserialize/objects.nim b/tests/json/deserialize/objects.nim index f295f91..93edeb9 100644 --- a/tests/json/deserialize/objects.nim +++ b/tests/json/deserialize/objects.nim @@ -99,7 +99,8 @@ suite "json - deserialize objects": myint: int let expected = MyRef(mystring: "abc", myint: 1) - let byteArray = """{ + let byteArray = + """{ "mystring": "abc", "myint": 1 }""".toBytes diff --git a/tests/json/testSerialize.nim b/tests/json/testSerialize.nim index c51dbc3..4338f34 100644 --- a/tests/json/testSerialize.nim +++ b/tests/json/testSerialize.nim @@ -50,7 +50,8 @@ suite "json serialization - serialize": let json = %*{"myobj": myobj, "mystuint": mystuint} - let expected = """{ + let expected = + """{ "myobj": { "mystring": "abc", "myint": 123, @@ -69,7 +70,8 @@ suite "json serialization - serialize": let obj = %MyObj(mystring: "abc", myint: 1, mybool: true) - let expected = """{ + let expected = + """{ "mystring": "abc", "myint": 1 }""".flatten @@ -83,7 +85,8 @@ suite "json serialization - serialize": let obj = %MyRef(mystring: "abc", myint: 1) - let expected = """{ + let expected = + """{ "mystring": "abc", "myint": 1 }""".flatten diff --git a/tests/test.nimble b/tests/test.nimble index 897430e..fc7114f 100644 --- a/tests/test.nimble +++ b/tests/test.nimble @@ -9,4 +9,3 @@ requires "questionable >= 0.10.13 & < 0.11.0" task test, "Run the test suite": exec "nimble install -d -y" exec "nim c -r test" - \ No newline at end of file From 0e2fd5c22d77026edc27cfc279af616c54225bc3 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Sat, 31 May 2025 17:57:34 +0530 Subject: [PATCH 13/20] refactor: rename encode to toCbor and toCbor to toCborNode for consistency --- serde/cbor/deserializer.nim | 99 ------------------------------------- serde/cbor/serializer.nim | 58 +++++++++++----------- tests/benchmark.nim | 18 +++---- tests/cbor/testObjects.nim | 67 ++++++++++++------------- 4 files changed, 72 insertions(+), 170 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 6007104..40fa119 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -593,105 +593,6 @@ proc fromCbor*[T: tuple](_: type T, n: CborNode): ?!T = success res -# proc fromCbor*[T](v: var T; n: CborNode): ?!void = -# try: -# when T is CborNode: -# v = n -# result = success() -# elif compiles(fromCborHook(v, n)): -# return fromCborHook(v, n) -# elif T is distinct: -# return fromCbor(distinctBase v, n) -# elif T is SomeUnsignedInt: -# expectCborKind(T, {cborUnsigned}, n) -# v = T n.uint -# if v.BiggestUInt == n.uint: -# return success() -# else: -# return failure(newCborError("Value overflow for unsigned integer")) -# elif T is SomeSignedInt: -# expectCborKind(T, {cborUnsigned, cborNegative}, n) -# if n.kind == cborUnsigned: -# v = T n.uint -# if v.BiggestUInt == n.uint: -# return success() -# else: -# return failure(newCborError("Value overflow for un signed integer")) -# elif n.kind == cborNegative: -# v = T n.int -# if v.BiggestInt == n.int: -# return success() -# else: -# return failure(newCborError("Value overflow for signed integer")) -# elif T is bool: -# if not n.isBool: -# return failure(newCborError("Expected boolean, got " & $n.kind)) -# v = n.getBool -# return success() -# elif T is SomeFloat: -# expectCborKind(T, {cborFloat}, n) -# v = T n.float -# return success() -# elif T is seq[byte]: -# expectCborKind(T, {cborBytes}, n) -# v = n.bytes -# return success() -# elif T is string: -# expectCborKind(T, {cborText}, n) -# v = n.text -# return success() -# elif T is seq: -# expectCborKind(T, {cborArray}, n) -# v.setLen n.seq.len -# for i, e in n.seq: -# let itemResult = fromCbor(v[i], e) -# if itemResult.isFailure: -# v.setLen 0 -# return failure(itemResult.error) -# return success() -# elif T is tuple: -# expectCborKind(T, {cborArray}, n) -# if n.seq.len != T.tupleLen: -# return failure(newCborError("Expected tuple of length " & $T.tupleLen)) -# var i: int -# for f in fields(v): -# let itemResult = fromCbor(f, n.seq[i]) -# if itemResult.isFailure: -# return failure(itemResult.error) -# inc i -# return success() -# elif T is ref: -# if n.isNull: -# v = nil -# return success() -# else: -# if isNil(v): new(v) -# return fromCbor(v[], n) -# elif T is object: -# expectCborKind(T, {cborMap}, n) -# var -# i: int -# key = CborNode(kind: cborText) -# for s, _ in fieldPairs(v): -# key.text = s -# if not n.map.hasKey key: -# return failure(newCborError("Missing field: " & s)) -# else: -# let fieldResult = fromCbor(v.dot(s), n.map[key]) -# if fieldResult.isFailure: -# return failure(fieldResult.error) -# inc i -# if i == n.map.len: -# return success() -# else: -# return failure(newCborError("Extra fields in map")) -# else: -# return failure(newCborError("Unsupported type: " & $T)) -# except CatchableError as e: -# return failure newCborError(e.msg) -# except Exception as e: -# raise newException(Defect, e.msg, e) - proc fromCbor*[T: ref](_: type T, n: CborNode): ?!T = when T is ref: if n.isNull: diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index a222f0d..ea0c241 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -58,20 +58,20 @@ proc writeInitial[T: SomeInteger](str: Stream, m: uint8, n: T): ?!void = str.write(n.uint8) elif uint64(n) <= uint64(uint16.high): str.write(m or 25'u8) - str.write((uint8) n shr 8) - str.write((uint8) n) + str.write((uint8)n shr 8) + str.write((uint8)n) elif uint64(n) <= uint64(uint32.high): str.write(m or 26'u8) for i in countdown(24, 8, 8): {.unroll.} - str.write((uint8) n shr i) - str.write((uint8) n) + str.write((uint8)n shr i) + str.write((uint8)n) else: str.write(m or 27'u8) for i in countdown(56, 8, 8): {.unroll.} - str.write((uint8) n shr i) - str.write((uint8) n) + str.write((uint8)n shr i) + str.write((uint8)n) success() except IOError as e: return failure(e.msg) @@ -198,20 +198,20 @@ proc writeCbor*[T: SomeFloat](str: Stream, v: T): ?!void = return success() of fcZero: str.write initialByte(7, 25) - str.write((char) 0x00) + str.write((char)0x00) of fcNegZero: str.write initialByte(7, 25) - str.write((char) 0x80) + str.write((char)0x80) of fcInf: str.write initialByte(7, 25) - str.write((char) 0x7c) + str.write((char)0x7c) of fcNan: str.write initialByte(7, 25) - str.write((char) 0x7e) + str.write((char)0x7e) of fcNegInf: str.write initialByte(7, 25) - str.write((char) 0xfc) - str.write((char) 0x00) + str.write((char)0xfc) + str.write((char)0x00) success() except IOError as io: return failure(io.msg) @@ -273,16 +273,16 @@ proc writeCbor*[T: object](str: Stream, v: T): ?!void = ?str.writeCbor(f) success() -proc writeCborArray*(str: Stream, args: varargs[CborNode, toCbor]): ?!void = +proc writeCborArray*(str: Stream, args: varargs[CborNode, toCborNode]): ?!void = ## Encode to a CBOR array in binary form. This magic doesn't ## always work, some arguments may need to be explicitly - ## converted with ``toCbor`` before passing. + ## converted with ``toCborNode`` before passing. ?str.writeCborArrayLen(args.len) for x in args: ?str.writeCbor(x) success() -proc encode*[T](v: T): ?!string = +proc toCbor*[T](v: T): ?!string = ## Encode an arbitrary value to CBOR binary representation. ## A wrapper over ``writeCbor``. let s = newStringStream() @@ -296,7 +296,7 @@ proc toRaw*(n: CborNode): ?!CborNode = if n.kind == cborRaw: return success(n) else: - without res =? encode(n), error: + without res =? toCbor(n), error: return failure(error) return success(CborNode(kind: cborRaw, raw: res)) @@ -345,25 +345,25 @@ proc writeCbor*(str: Stream, t: Time): ?!void = ?writeCbor(str, t.toUnix) success() -func toCbor*(x: CborNode): ?!CborNode = +func toCborNode*(x: CborNode): ?!CborNode = success(x) -func toCbor*(x: SomeInteger): ?!CborNode = +func toCborNode*(x: SomeInteger): ?!CborNode = if x > 0: success(CborNode(kind: cborUnsigned, uint: x.uint64)) else: success(CborNode(kind: cborNegative, int: x.int64)) -func toCbor*(x: openArray[byte]): ?!CborNode = +func toCborNode*(x: openArray[byte]): ?!CborNode = success(CborNode(kind: cborBytes, bytes: @x)) -func toCbor*(x: string): ?!CborNode = +func toCborNode*(x: string): ?!CborNode = success(CborNode(kind: cborText, text: x)) -func toCbor*(x: openArray[CborNode]): ?!CborNode = +func toCborNode*(x: openArray[CborNode]): ?!CborNode = success(CborNode(kind: cborArray, seq: @x)) -func toCbor*(pairs: openArray[(CborNode, CborNode)]): ?!CborNode = +func toCborNode*(pairs: openArray[(CborNode, CborNode)]): ?!CborNode = try: return success(CborNode(kind: cborMap, map: pairs.toOrderedTable)) except CatchableError as e: @@ -371,24 +371,24 @@ func toCbor*(pairs: openArray[(CborNode, CborNode)]): ?!CborNode = except Exception as e: raise newException(Defect, e.msg, e) -func toCbor*(tag: uint64, val: CborNode): ?!CborNode = - without res =? toCbor(val), error: +func toCborNode*(tag: uint64, val: CborNode): ?!CborNode = + without res =? toCborNode(val), error: return failure(error.msg) var cnode = res cnode.tag = some(tag) return success(cnode) -func toCbor*(x: bool): ?!CborNode = +func toCborNode*(x: bool): ?!CborNode = case x of false: success(CborNode(kind: cborSimple, simple: 20)) of true: success(CborNode(kind: cborSimple, simple: 21)) -func toCbor*(x: SomeFloat): ?!CborNode = +func toCborNode*(x: SomeFloat): ?!CborNode = success(CborNode(kind: cborFloat, float: x.float64)) -func toCbor*(x: pointer): ?!CborNode = +func toCborNode*(x: pointer): ?!CborNode = ## A hack to produce a CBOR null item. if not x.isNil: return failure("pointer is not nil") @@ -398,7 +398,7 @@ func initCborBytes*[T: char | byte](buf: openArray[T]): CborNode = ## Create a CBOR byte string from `buf`. result = CborNode(kind: cborBytes, bytes: newSeq[byte](buf.len)) for i in 0 ..< buf.len: - result.bytes[i] = (byte) buf[i] + result.bytes[i] = (byte)buf[i] func initCborBytes*(len: int): CborNode = ## Create a CBOR byte string of ``len`` bytes. @@ -421,6 +421,6 @@ func initCborMap*(initialSize = tables.defaultInitialSize): CborNode = ## Initialize a CBOR map. CborNode(kind: cborMap, map: initOrderedTable[CborNode, CborNode](initialSize)) -func initCbor*(items: varargs[CborNode, toCbor]): CborNode = +func initCbor*(items: varargs[CborNode, toCborNode]): CborNode = ## Initialize a CBOR arrary. CborNode(kind: cborArray, seq: @items) diff --git a/tests/benchmark.nim b/tests/benchmark.nim index 32f83c5..9be16b2 100644 --- a/tests/benchmark.nim +++ b/tests/benchmark.nim @@ -13,14 +13,14 @@ type Inner {.serialize.} = object size: uint64 type CustomPoint {.serialize.} = object - u: uint64 # Unsigned integer - n: int # Signed integer - b: seq[byte] # Byte sequence - t: string # Text string - arr: seq[int] # Integer sequence - tag: float # Floating point - flag: bool # Boolean - inner: Inner # Nested object + u: uint64 # Unsigned integer + n: int # Signed integer + b: seq[byte] # Byte sequence + t: string # Text string + arr: seq[int] # Integer sequence + tag: float # Floating point + flag: bool # Boolean + inner: Inner # Nested object innerArr: seq[Inner] # Sequence of objects proc generateCustomPoint(): CustomPoint = @@ -49,7 +49,7 @@ proc benchmark(): void = let cborStartTime = cpuTime() for i in 1 .. 100000: - cborStr = encode(point).tryValue + cborStr = toCbor(point).tryValue let cborEndTime = cpuTime() let cborDuration = cborEndTime - cborStartTime diff --git a/tests/cbor/testObjects.nim b/tests/cbor/testObjects.nim index 20be59a..53bcc71 100644 --- a/tests/cbor/testObjects.nim +++ b/tests/cbor/testObjects.nim @@ -48,22 +48,22 @@ type # Complex object with various field types to test comprehensive serialization CompositeNested = object - u: uint64 # Unsigned integer - n: int # Signed integer - b: seq[byte] # Byte sequence - t: string # Text string - arr: seq[int] # Integer sequence - tag: float # Floating point - flag: bool # Boolean - inner: Inner # Nested object - innerArr: seq[Inner] # Sequence of objects + u: uint64 # Unsigned integer + n: int # Signed integer + b: seq[byte] # Byte sequence + t: string # Text string + arr: seq[int] # Integer sequence + tag: float # Floating point + flag: bool # Boolean + inner: Inner # Nested object + innerArr: seq[Inner] # Sequence of objects coordinates: tuple[x: int, y: int, label: string] # Tuple - refInner: ref Inner # Reference to object - refNewInner: NewType # Custom reference type - refNil: ref Inner # Nil reference - customPoint: CustomPoint # Custom type - time: Time # Time - date: DateTime # DateTime + refInner: ref Inner # Reference to object + refNewInner: NewType # Custom reference type + refNil: ref Inner # Nil reference + customPoint: CustomPoint # Custom type + time: Time # Time + date: DateTime # DateTime # Custom deserialization for CustomColor enum # Converts a CBOR negative integer to a CustomColor enum value @@ -107,7 +107,8 @@ proc createPointCbor(x, y: int): CborNode = ] # Creates a CBOR map node representing a CustomObject -proc createObjectCbor(name: string, point: CustomPoint, color: CustomColor): CborNode = +proc createObjectCbor(name: string, point: CustomPoint, + color: CustomColor): CborNode = result = CborNode(kind: cborMap) result.map = initOrderedTable[CborNode, CborNode]() @@ -156,29 +157,29 @@ suite "CBOR deserialization": # 2. Create a complex object with all supported types var original = CompositeNested( - u: 42, # unsigned integer - n: -99, # signed integer - b: @[byte 1, byte 2], # byte array - t: "hi", # string - arr: @[1, 2, 3], # integer array - tag: 1.5, # float - flag: true, # boolean - inner: Inner(s: "inner!", nums: @[10, 20]), # nested object + u: 42, # unsigned integer + n: -99, # signed integer + b: @[byte 1, byte 2], # byte array + t: "hi", # string + arr: @[1, 2, 3], # integer array + tag: 1.5, # float + flag: true, # boolean + inner: Inner(s: "inner!", nums: @[10, 20]), # nested object innerArr: - @[ # array of objects + @[ # array of objects Inner(s: "first", nums: @[1, 2]), Inner(s: "second", nums: @[3, 4, 5]) ], - coordinates: (x: 10, y: 20, label: "test"), # tuple - refInner: refInner, # reference to object - refNewInner: refNewObj, # custom reference type - refNil: nil, # nil reference - customPoint: CustomPoint(x: 15, y: 25), # custom type - time: getTime(), # time - date: now().utc, # date + coordinates: (x: 10, y: 20, label: "test"), # tuple + refInner: refInner, # reference to object + refNewInner: refNewObj, # custom reference type + refNil: nil, # nil reference + customPoint: CustomPoint(x: 15, y: 25), # custom type + time: getTime(), # time + date: now().utc, # date ) # Test serialization using encode helper - without encodedStr =? encode(original), error: + without encodedStr =? toCbor(original), error: fail() # Test serialization using stream API From a4cd9a064dd74318a56122509f7d64dd79c82e80 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Sat, 31 May 2025 17:57:56 +0530 Subject: [PATCH 14/20] test changes --- tests/cbor/testPrimitives.nim | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/cbor/testPrimitives.nim b/tests/cbor/testPrimitives.nim index 945d4b4..38a7f75 100644 --- a/tests/cbor/testPrimitives.nim +++ b/tests/cbor/testPrimitives.nim @@ -49,7 +49,7 @@ suite "roundtrip": controlCbor = base64.decode controlB64 name = v["name"].getStr test name: - without testCbor =? encode(parseCbor(controlCbor)), error: + without testCbor =? toCbor(parseCbor(controlCbor)), error: fail() if controlCbor != testCbor: let testB64 = base64.encode(testCbor) @@ -59,16 +59,16 @@ suite "hooks": test "DateTime": let dt = now() - without bin =? encode(dt), error: + without bin =? toCbor(dt), error: fail() check(parseCbor(bin).text == $dt) test "Time": let t = now().toTime - var bin = encode(t).tryValue + var bin = toCbor(t).tryValue check(parseCbor(bin).getInt == t.toUnix) test "tag": - var c = toCbor("foo").tryValue + var c = toCborNode("foo").tryValue c.tag = some(99'u64) check c.tag == some(99'u64) @@ -76,19 +76,19 @@ test "sorting": var map = initCborMap() var keys = @[ - toCbor(10).tryValue, - toCbor(100).tryValue, - toCbor(-1).tryValue, - toCbor("z").tryValue, - toCbor("aa").tryValue, - toCbor([toCbor(100).tryValue]).tryValue, - toCbor([toCbor(-1).tryValue]).tryValue, - toCbor(false).tryValue, + toCborNode(10).tryValue, + toCborNode(100).tryValue, + toCborNode(-1).tryValue, + toCborNode("z").tryValue, + toCborNode("aa").tryValue, + toCborNode([toCborNode(100).tryValue]).tryValue, + toCborNode([toCborNode(-1).tryValue]).tryValue, + toCborNode(false).tryValue, ] shuffle(keys) for k in keys: - map[k] = toCbor(0).tryValue + map[k] = toCborNode(0).tryValue check not map.isSorted.tryValue check sort(map).isSuccess check map.isSorted.tryValue From 50000bc0c91eb3b788fd2424fcf25e258cd10061 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Sat, 31 May 2025 19:00:18 +0530 Subject: [PATCH 15/20] refactor: convert parseAssert to template and fix line wrapping in CBOR deserializer --- serde/cbor/deserializer.nim | 11 ++++------- serde/cbor/errors.nim | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 40fa119..da1b40b 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -160,8 +160,7 @@ func textLen*(c: CborParser): int {.raises: [CborParseError].} = parseAssert(c.kind == CborEventKind.cborText, "Expected text, got " & $c.kind) c.intVal.int -proc nextText*(c: var CborParser, buf: var string) {.raises: [ - CborParseError].} = +proc nextText*(c: var CborParser, buf: var string) {.raises: [CborParseError].} = ## Read the text that the parser is positioned on into a string and advance. try: parseAssert(c.kind == CborEventKind.cborText, "Expected text, got " & $c.kind) @@ -200,8 +199,7 @@ proc skipNode*(c: var CborParser) {.raises: [CborParseError].} = case c.kind of CborEventKind.cborEof: raise newCborError("end of CBOR stream") - of CborEventKind.cborPositive, CborEventKind.cborNegative, - CborEventKind.cborSimple: + of CborEventKind.cborPositive, CborEventKind.cborNegative, CborEventKind.cborSimple: c.next() of CborEventKind.cborBytes, CborEventKind.cborText: if c.isIndefinite: @@ -312,8 +310,7 @@ proc nextNode*(c: var CborParser): CborNode {.raises: [CborParseError].} = of CborEventKind.cborMap: let mapLen = c.intVal.int result = CborNode( - kind: cborMap, map: initOrderedTable[CborNode, CborNode]( - mapLen.nextPowerOfTwo) + kind: cborMap, map: initOrderedTable[CborNode, CborNode](mapLen.nextPowerOfTwo) ) if c.isIndefinite: c.next() @@ -486,7 +483,7 @@ func isTagged*(n: CborNode): bool = func hasTag*(n: CborNode, tag: Natural): bool = ## Check if a CBOR item has a tag. - n.tag.isSome and n.tag.get == (uint64)tag + n.tag.isSome and n.tag.get == (uint64) tag proc `tag=`*(result: var CborNode, tag: Natural) = ## Tag a CBOR item. diff --git a/serde/cbor/errors.nim b/serde/cbor/errors.nim index e028312..84d4c6f 100644 --- a/serde/cbor/errors.nim +++ b/serde/cbor/errors.nim @@ -37,6 +37,6 @@ proc newUnexpectedKindError*( proc newCborError*(msg: string): ref CborParseError = newException(CborParseError, msg) -proc parseAssert*(check: bool, msg = "") {.inline.} = +template parseAssert*(check: bool, msg = "") = if not check: raise newException(CborParseError, msg) From 0096796d660cbb4a7f176d41edb660cd79cddd60 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Sat, 31 May 2025 20:36:49 +0530 Subject: [PATCH 16/20] add compile-time checks to prevent use of serialize/deserialize pragmas in CBOR --- serde/cbor/deserializer.nim | 7 +++-- serde/cbor/errors.nim | 4 --- serde/cbor/helpers.nim | 9 +++++++ serde/cbor/serializer.nim | 7 +++++ tests/cbor/testObjects.nim | 3 --- tests/cbor/testPragmaChecks.nim | 45 +++++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 tests/cbor/testPragmaChecks.nim diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index da1b40b..64c67a7 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -607,10 +607,11 @@ proc fromCbor*[T: object](_: type T, n: CborNode): ?!T = return success T(n) expectCborKind(T, {cborMap}, n) - var res = T.default - let mode = getSerdeMode(T, deserialize) + # Added because serde {serialize, deserialize} pragmas and options are not supported cbor + assertNoPragma(T, deserialize, "deserialize pragma not supported") + try: var i: int @@ -621,6 +622,8 @@ proc fromCbor*[T: object](_: type T, n: CborNode): ?!T = else: res ): + assertNoPragma(value, deserialize, "deserialize pragma not supported") + key.text = name if not n.map.hasKey key: diff --git a/serde/cbor/errors.nim b/serde/cbor/errors.nim index 84d4c6f..20e9751 100644 --- a/serde/cbor/errors.nim +++ b/serde/cbor/errors.nim @@ -36,7 +36,3 @@ proc newUnexpectedKindError*( proc newCborError*(msg: string): ref CborParseError = newException(CborParseError, msg) - -template parseAssert*(check: bool, msg = "") = - if not check: - raise newException(CborParseError, msg) diff --git a/serde/cbor/helpers.nim b/serde/cbor/helpers.nim index 9810c6c..dc26430 100644 --- a/serde/cbor/helpers.nim +++ b/serde/cbor/helpers.nim @@ -27,6 +27,15 @@ template expectCborKind*( ) = expectCborKind(expectedType, {expectedKind}, cbor) +template parseAssert*(check: bool, msg = "") = + if not check: + raise newException(CborParseError, msg) + +template assertNoPragma*(value, pragma, msg) = + static: + when value.hasCustomPragma(pragma): + raiseAssert(msg) + macro dot*(obj: object, fld: string): untyped = ## Turn ``obj.dot("fld")`` into ``obj.fld``. newDotExpr(obj, newIdentNode(fld.strVal)) diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index ea0c241..bdfefb6 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -5,7 +5,9 @@ import std/[streams, options, tables, typetraits, math, endians, times] import pkg/questionable import pkg/questionable/results import ../utils/errors +import ../utils/pragmas import ./types +import ./helpers {.push raises: [].} @@ -265,10 +267,15 @@ proc writeCbor*(str: Stream, v: CborNode): ?!void = proc writeCbor*[T: object](str: Stream, v: T): ?!void = var n: uint + # Added because serde {serialize, deserialize} pragma and options are not supported cbor + assertNoPragma(T, serialize, "serialize pragma not supported") + for _, _ in v.fieldPairs: inc n ?str.writeInitial(5, n) + for k, f in v.fieldPairs: + assertNoPragma(f, serialize, "serialize pragma not supported") ?str.writeCbor(k) ?str.writeCbor(f) success() diff --git a/tests/cbor/testObjects.nim b/tests/cbor/testObjects.nim index 53bcc71..8c86aa0 100644 --- a/tests/cbor/testObjects.nim +++ b/tests/cbor/testObjects.nim @@ -43,9 +43,6 @@ type NewType = ref object size: uint64 - # Reference type for Inner object - InnerRef = ref Inner - # Complex object with various field types to test comprehensive serialization CompositeNested = object u: uint64 # Unsigned integer diff --git a/tests/cbor/testPragmaChecks.nim b/tests/cbor/testPragmaChecks.nim new file mode 100644 index 0000000..48ab0ef --- /dev/null +++ b/tests/cbor/testPragmaChecks.nim @@ -0,0 +1,45 @@ +import std/unittest +import std/streams + +import ../../serde/cbor +import ../../serde/utils/pragmas + +{.push raises: [].} + +suite "CBOR pragma checks": + test "fails to compile when object marked with 'serialize' pragma": + type SerializeTest {.serialize.} = object + value: int + + check not compiles(toCbor(SerializeTest(value: 42))) + + test "fails to compile when object marked with 'deserialize' pragma": + type DeserializeTest {.deserialize.} = object + value: int + + let node = CborNode(kind: cborMap) + check not compiles(DeserializeTest.fromCbor(node)) + + test "fails to compile when field marked with 'serialize' pragma": + type FieldSerializeTest = object + normalField: int + pragmaField {.serialize.}: int + + check not compiles(toCbor(FieldSerializeTest(normalField: 42, pragmaField: 100))) + + test "fails to compile when field marked with 'deserialize' pragma": + type FieldDeserializeTest = object + normalField: int + pragmaField {.deserialize.}: int + + let node = CborNode(kind: cborMap) + check not compiles(FieldDeserializeTest.fromCbor(node)) + + test "compiles when type has no pragmas": + type NoPragmaTest = object + value: int + + check compiles(toCbor(NoPragmaTest(value: 42))) + + let node = CborNode(kind: cborMap) + check compiles(NoPragmaTest.fromCbor(node)) From dd000984664d0b188cdc3fc2820448055f6f941c Mon Sep 17 00:00:00 2001 From: munna0908 Date: Mon, 2 Jun 2025 00:12:31 +0530 Subject: [PATCH 17/20] update readme and add docs for cbor and json wireformats --- README.md | 287 +----------------------- serde/cbor/README.md | 376 ++++++++++++++++++++++++++++++++ serde/cbor/deserializer.nim | 2 +- serde/cbor/serializer.nim | 28 +-- serde/json/README.md | 421 ++++++++++++++++++++++++++++++++++++ 5 files changed, 822 insertions(+), 292 deletions(-) create mode 100644 serde/cbor/README.md create mode 100644 serde/json/README.md diff --git a/README.md b/README.md index 1b0d338..1660ecc 100644 --- a/README.md +++ b/README.md @@ -1,102 +1,13 @@ # nim-serde -Easy-to-use json serialization capabilities, and a drop-in replacement for `std/json`. +A serialization and deserialization library for Nim supporting multiple wire formats. -## Quick examples +## Supported Wire Formats -Opt-in serialization by default: +nim-serde currently supports the following wire formats: -```nim -import pkg/serde/json - -type MyType = object - field1 {.serialize.}: bool - field2: bool - -assert MyType(field1: true, field2: true).toJson == """{"field1":true}""" -``` - -Opt-out deserialization by default: - -```nim -import pkg/serde/json - -# All fields deserialized, as none are ignored -type MyType1 = object - field1: bool - field2: bool - -let jsn1 = """{ - "field1": true, - "field2": true - }""" -assert !MyType1.fromJson(jsn1) == MyType1(field1: true, field2: true) - -# Don't deserialize ignored fields in OptOut mode -type MyType2 = object - field1 {.deserialize(ignore=true).}: bool - field2: bool - -let jsn2 = """{ - "field1": true, - "field2": true, - "extra": "extra fields don't error in OptOut mode" - }""" -assert !MyType2.fromJson(jsn2) == MyType2(field1: false, field2: true) - -# Note, the ! operator is part of https://github.com/codex-storage/questionable, which retrieves a value if set -``` - -Serialize all fields of a type (OptOut mode): - -```nim -import pkg/serde/json - -type MyType {.serialize.} = object - field1: int - field2: int - -assert MyType(field1: 1, field2: 2).toJson == """{"field1":1,"field2":2}""" -``` - -Alias field names in both directions! - -```nim -import pkg/serde/json - -type MyType {.serialize.} = object - field1 {.serialize("othername"),deserialize("takesprecedence").}: int - field2: int - -assert MyType(field1: 1, field2: 2).toJson == """{"othername":1,"field2":2}""" -let jsn = """{ - "othername": 1, - "field2": 2, - "takesprecedence": 3 - }""" -assert !MyType.fromJson(jsn) == MyType(field1: 3, field2: 2) -``` - -Supports strict mode, where type fields and json fields must match - -```nim -import pkg/serde/json - -type MyType {.deserialize(mode=Strict).} = object - field1: int - field2: int - -let jsn = """{ - "field1": 1, - "field2": 2, - "extra": 3 - }""" - -let res = MyType.fromJson(jsn) -assert res.isFailure -assert res.error of SerdeError -assert res.error.msg == "json field(s) missing in object: {\"extra\"}" -``` +- **JSON**: A text-based data interchange format. [See JSON](serde/json/README.md) for details. +- **CBOR**: A binary data format following RFC 8949. [See CBOR](serde/cbor/README.md) for details. ## Serde modes @@ -215,198 +126,20 @@ Specifying `ignore`, will prevent de/serialization on the field. | `ignore` |
  • **OptOut:** field not serialized
  • **OptIn:** field not serialized
  • **Strict:** field serialized
    • |
  • **OptOut:** field not deserialized
  • **OptIn:** field not deserialized
  • **Strict:** field deserialized
    • | -## Deserialization -`serde` deserializes using `fromJson`, and in all instances returns `Result[T, -CatchableError]`, where `T` is the type being deserialized. For example: -```nim -type MyType = object - field1: bool - field2: bool +## Known Issues -let jsn1 = """{ - "field1": true, - "field2": true - }""" +There is a known issue when using mixins with generic overloaded procs like `fromJson`. At the time of mixin call, only the `fromJson` overloads in scope of the called mixin are available to be dispatched at runtime. There could be other `fromJson` overloads declared in other modules, but are not in scope at the time the mixin was called. -assert !MyType.fromJson(jsn1) == MyType(field1: true, field2: true) -``` +Therefore, anytime `fromJson` is called targeting a declared overload, it may or may not be dispatchable. This can be worked around by forcing the `fromJson` overload into scope at compile time. For example, in your application where the `fromJson` overload is defined, at the bottom of the module add: -If there was an error during deserialization, the result of `fromJson` will contain it: - -```nim -import pkg/serde/json - -type MyType {.deserialize(mode=Strict).} = object - field1: int - field2: int - -let jsn = """{ - "field1": 1, - "field2": 2, - "extra": 3 - }""" - -let res = MyType.fromJson(jsn) -assert res.isFailure -assert res.error of SerdeError -assert res.error.msg == "json field(s) missing in object: {\"extra\"}" -``` - -## Custom types - -If `serde` can't de/serialize a custom type, de/serialization can be supported by -overloading `%` and `fromJson`. For example: - -```nim -type - Address* = distinct array[20, byte] - SerializationError* = object of CatchableError - -func `%`*(address: Address): JsonNode = - %($address) - -func fromJson(_: type Address, json: JsonNode): ?!Address = - expectJsonKind(Address, JString, json) - without address =? Address.init(json.getStr), error: - return failure newException(SerializationError, - "Failed to convert '" & $json & "' to Address: " & error.msg) - success address -``` - -## Serializing to string (`toJson`) - -`toJson` is a shortcut for serializing an object into its serialized string -representation: - -```nim -import pkg/serde/json - -type MyType {.serialize.} = object - field1: string - field2: bool - -let mt = MyType(field1: "hw", field2: true) -assert mt.toJson == """{"field1":"hw","field2":true}""" -``` - -This comes in handy, for example, when sending API responses: - -```nim -let availability = getAvailability(...) -return RestApiResponse.response(availability.toJson, - contentType="application/json") -``` - -## `std/json` drop-in replacment - -`nim-serde` can be used as a drop-in replacement for the [standard library's `json` -module](https://nim-lang.org/docs/json.html), with a few notable improvements. - -Instead of importing `std/json` into your application, `pkg/serde/json` can be imported -instead: - -```diff -- import std/json -+ import pkg/serde/json -``` - -As with `std/json`, `%` can be used to serialize a type into a `JsonNode`: - -```nim -import pkg/serde/json - -assert %"hello" == newJString("hello") -``` - -And `%*` can be used to serialize objects: - -```nim -import pkg/serde/json - -let expected = newJObject() -expected["hello"] = newJString("world") -assert %*{"hello": "world"} == expected -``` - -As well, serialization of types can be overridden, and serialization of custom types can -be introduced. Here, we are overriding the serialization of `int`: - -```nim -import pkg/serde/json - -func `%`(i: int): JsonNode = - newJInt(i + 1) - -assert 1.toJson == "2" -``` - -## `parseJson` and exception tracking - -Unfortunately, `std/json`'s `parseJson` can raise an `Exception`, so proper exception -tracking breaks, eg - -```nim - -## Fails to compile: -## Error: parseJson(me, false, false) can raise an unlisted exception: Exception - -import std/json - -{.push raises:[].} - -type - MyAppError = object of CatchableError - -proc parseMe(me: string): JsonNode = - try: - return me.parseJson - except CatchableError as error: - raise newException(MyAppError, error.msg) - -assert """{"hello":"world"}""".parseMe == %* { "hello": "world" } -``` - -This is due to `std/json`'s `parseJson` incorrectly raising `Exception`. This can be -worked around by instead importing `serde` and calling its `JsonNode.parse` routine. -Note that `serde`'s `JsonNode.parse` returns a `Result[JsonNode, CatchableError]` -instead of just a plain `JsonNode` object as in `std/json`'s `parseJson`: - -```nim -import pkg/serde/json - -{.push raises:[].} - -type - MyAppError = object of CatchableError - -proc parseMe(me: string): JsonNode {.raises: [MyAppError].} = - without parsed =? JsonNode.parse(me), error: - raise newException(MyAppError, error.msg) - parsed - -assert """{"hello":"world"}""".parseMe == %* { "hello": "world" } -``` - -## Known issues - -There is a known issue when using mixins with generic overloaded procs like -`fromJson`. At the time of mixin call, only the `fromJson` overloads in scope of -the called mixin are available to be dispatched at runtime. There could be other -`fromJson` overloads declared in other modules, but are not in scope at the time -the mixin was called. Therefore, anytime `fromJson` is called targeting a -declared overload, it may or may not be dispatchable. This can be worked around -by forcing the `fromJson` overload into scope at compile time. For example, in -your application where the `fromJson` overload is defined, at the bottom of the -module add: ```nim static: MyType.fromJson("") ``` + This will ensure that the `MyType.fromJson` overload is dispatchable. -The basic types that serde supports should already have their overloads forced -in scope in [the `deserializer` -module](./serde/json/deserializer.nim#L340-L356). +The basic types that serde supports should already have their overloads forced in scope in [the `deserializer` module](./serde/json/deserializer.nim#L340-L356). For an illustration of the problem, please see this [narrow example](https://github.com/gmega/serialization-bug/tree/main/narrow) by [@gmega](https://github.com/gmega). diff --git a/serde/cbor/README.md b/serde/cbor/README.md new file mode 100644 index 0000000..f546c4d --- /dev/null +++ b/serde/cbor/README.md @@ -0,0 +1,376 @@ +# nim-serde CBOR + +This README details the usage of CBOR serialization and deserialization features offered by nim-serde, in compliance with [RFC 8949](https://datatracker.ietf.org/doc/html/rfc8949). + +## Table of Contents +- [nim-serde CBOR](#nim-serde-cbor) + - [Table of Contents](#table-of-contents) + - [Serialization API](#serialization-api) + - [Basic Serialization with Stream API](#basic-serialization-with-stream-api) + - [Object Serialization](#object-serialization) + - [Custom Type Serialization](#custom-type-serialization) + - [Converting to CBOR with `toCbor`](#converting-to-cbor-with-tocbor) + - [Working with CborNode](#working-with-cbornode) + - [Convenience Functions for CborNode](#convenience-functions-for-cbornode) + - [Deserialization API](#deserialization-api) + - [Basic Deserialization with `fromCbor`](#basic-deserialization-with-fromcbor) + - [Error Handling](#error-handling) + - [Parsing CBOR with `parseCbor`](#parsing-cbor-with-parsecbor) + - [Custom Type Deserialization](#custom-type-deserialization) + - [Implementation Details](#implementation-details) + - [Current Limitations](#current-limitations) + +## Serialization API + +The nim-serde CBOR serialization API provides several ways to convert Nim values to CBOR. + +### Basic Serialization with Stream API + +The `writeCbor` function writes Nim values to a stream in CBOR format: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/streams + +# Create a stream to write to +let stream = newStringStream() + +# Basic types +discard stream.writeCbor(42) # Unsigned integer +discard stream.writeCbor(-10) # Negative integer +discard stream.writeCbor(3.14) # Float +discard stream.writeCbor("hello") # String +discard stream.writeCbor(true) # Boolean + +# Arrays and sequences +discard stream.writeCbor(@[1, 2, 3]) # Sequence + +# Get the serialized CBOR data +let cborData = stream.data +``` + +### Object Serialization + +Objects can be serialized to CBOR format using the stream API: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/streams + +type Person = object + name: string + age: int + isActive: bool + +let person = Person( + name: "John", + age: 30, + isActive: true +) + +# Serialize the object to CBOR +let stream = newStringStream() +discard stream.writeCbor(person) + +# Get the serialized CBOR data +let cborData = stream.data +``` + +### Custom Type Serialization + +You can extend nim-serde to support custom types by defining your own `writeCbor` procs: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/streams +import std/strutils + +# Define a custom type +type + UserId = distinct int + +# Custom serialization for UserId +proc writeCbor*(str: Stream, id: UserId): ?!void = + # Write as a CBOR text string with a prefix + str.writeCbor("user-" & $int(id)) + +# Test serialization +let userId = UserId(42) +let stream = newStringStream() +discard stream.writeCbor(userId) +let cborData = stream.data + +# Test in object context +type User = object + id: UserId + name: string + +let user = User(id: UserId(123), name: "John") +let userStream = newStringStream() +discard userStream.writeCbor(user) +let userCborData = userStream.data +``` + +### Converting to CBOR with `toCbor` + +The `toCbor` function can be used to directly convert a Nim value to CBOR binary data: + +```nim +import pkg/serde/cbor +import pkg/questionable/results + +type Person = object + name: string + age: int + isActive: bool + +let person = Person( + name: "John", + age: 30, + isActive: true +) + +# Convert to CBOR binary data +let result = toCbor(person) +assert result.isSuccess +let cborData = !result +``` + +### Working with CborNode + +The `CborNode` type represents CBOR data in memory and can be manipulated directly: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/tables + +# Create CBOR nodes +let textNode = CborNode(kind: cborText, text: "hello") +let intNode = CborNode(kind: cborUnsigned, uint: 42'u64) +let floatNode = CborNode(kind: cborFloat, float: 3.14) + +# Create an array +var arrayNode = CborNode(kind: cborArray) +arrayNode.seq = @[textNode, intNode, floatNode] + +# Create a map with text keys and boolean values +var mapNode = CborNode(kind: cborMap) +mapNode.map = initOrderedTable[CborNode, CborNode]() +# Boolean values are represented as simple values (21 for true, 20 for false) +mapNode.map[CborNode(kind: cborText, text: "a")] = CborNode(kind: cborSimple, simple: 21) # true +mapNode.map[CborNode(kind: cborText, text: "b")] = CborNode(kind: cborSimple, simple: 20) # false + +# Convert to CBOR binary data +let result = toCbor(mapNode) +assert result.isSuccess +let cborData = !result +``` + +### Convenience Functions for CborNode + +The library provides convenience functions for creating CBOR nodes: + +```nim +import pkg/serde/cbor +import pkg/questionable/results + +# Initialize CBOR nodes +let bytesNode = initCborBytes(@[byte 1, byte 2, byte 3]) +let textNode = initCborText("hello") +let arrayNode = initCborArray() +let mapNode = initCborMap() + +# Convert values to CborNode +let intNodeResult = toCborNode(42) +assert intNodeResult.isSuccess +let intNode = !intNodeResult + +let strNodeResult = toCborNode("hello") +assert strNodeResult.isSuccess +let strNode = !strNodeResult + +let boolNodeResult = toCborNode(true) +assert boolNodeResult.isSuccess +let boolNode = !boolNodeResult +``` + +## Deserialization API + +The nim-serde CBOR deserialization API provides ways to convert CBOR data back to Nim values. + +### Basic Deserialization with `fromCbor` + +The `fromCbor` function converts CBOR data to Nim values: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/streams + +# Create some CBOR data +let stream = newStringStream() +discard stream.writeCbor(42) +let cborData = stream.data + +# Parse the CBOR data into a CborNode +try: + let node = parseCbor(cborData) + + # Deserialize the CborNode to a Nim value + let intResult = int.fromCbor(node) + assert intResult.isSuccess + let value = !intResult + assert value == 42 + + # You can also deserialize to other types + # For example, if cborData contained a string: + # let strResult = string.fromCbor(node) + # assert strResult.isSuccess + # let strValue = !strResult + +# Deserialize to an object +type Person = object + name: string + age: int + isActive: bool + +let personResult = Person.fromCbor(node) +assert personResult.isSuccess +let person = !personResult + +# Verify the deserialized data +assert person.name == "John" +assert person.age == 30 +assert person.isActive == true +``` + +### Error Handling + +Deserialization returns a `Result` type from the `questionable` library, allowing for safe error handling: + +```nim +import pkg/serde/cbor +import pkg/questionable/results + +# Invalid CBOR data for an integer +let invalidNode = CborNode(kind: cborText, text: "not an int") +let result = int.fromCbor(invalidNode) + +# Check for failure +assert result.isFailure +echo result.error.msg +# Output: "deserialization to int failed: expected {cborUnsigned, cborNegative} but got cborText" +``` + +### Parsing CBOR with `parseCbor` + +The `parseCbor` function parses CBOR binary data into a `CborNode`: + +```nim +import pkg/serde/cbor +import pkg/questionable/results + +# Parse CBOR data +let node = parseCbor(cborData) + +# Check node type and access data +case node.kind +of cborUnsigned: + echo "Unsigned integer: ", node.uint +of cborNegative: + echo "Negative integer: ", node.int +of cborText: + echo "Text: ", node.text +of cborArray: + echo "Array with ", node.seq.len, " items" +of cborMap: + echo "Map with ", node.map.len, " pairs" +else: + echo "Other CBOR type: ", node.kind +``` + +### Custom Type Deserialization + +You can extend nim-serde to support custom type deserialization by defining your own `fromCbor` procs: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/strutils + +# Define a custom type +type + UserId = distinct int + +# Custom deserialization for UserId +proc fromCbor*(_: type UserId, n: CborNode): ?!UserId = + if n.kind != cborText: + return failure(newSerdeError("Expected string for UserId, got " & $n.kind)) + + let str = n.text + if str.startsWith("user-"): + let idStr = str[5..^1] + try: + let id = parseInt(idStr) + success(UserId(id)) + except ValueError: + failure(newSerdeError("Invalid UserId format: " & str)) + else: + failure(newSerdeError("UserId must start with 'user-' prefix")) + +# Test deserialization +let node = parseCbor(cborData) # Assuming cborData contains a serialized UserId +let result = UserId.fromCbor(node) +assert result.isSuccess +assert int(!result) == 42 + +# Test deserialization in object context +type User = object + id: UserId + name: string + +let userNode = parseCbor(userCborData) # Assuming userCborData contains a serialized User +let userResult = User.fromCbor(userNode) +assert userResult.isSuccess +assert int((!userResult).id) == 123 +assert (!userResult).name == "John" +``` + + +## Implementation Details + +The CBOR serialization in nim-serde follows a stream-based approach: + +``` +# Serialization flow +Nim value → writeCbor → CBOR binary data + +# Deserialization flow +CBOR binary data → parseCbor (CborNode) → fromCbor → Nim value +``` + +Unlike the JSON implementation which uses the `%` operator pattern, the CBOR implementation uses a hook-based approach: + +1. The `writeCbor` function writes Nim values directly to a stream in CBOR format +2. Custom types can be supported by defining `writeCbor` procs for those types +3. The `toCbor` function provides a convenient way to convert values to CBOR binary data + +For deserialization, the library parses CBOR data into a `CborNode` representation, which can then be converted to Nim values using the `fromCbor` function. This approach allows for flexible handling of CBOR data while maintaining type safety. + +### Current Limitations + +While the JSON implementation supports serde modes via pragmas, the current CBOR implementation does not support the `serialize` and `deserialize` pragmas. The library will raise an assertion error if you try to use these pragmas with CBOR serialization. + +```nim +import pkg/serde/cbor + +type Person {.serialize(mode = OptOut).} = object # This will raise an assertion error + name: string + age: int + isActive: bool +``` + diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 64c67a7..55470c1 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -609,7 +609,7 @@ proc fromCbor*[T: object](_: type T, n: CborNode): ?!T = expectCborKind(T, {cborMap}, n) var res = T.default - # Added because serde {serialize, deserialize} pragmas and options are not supported cbor + # Added because serde {serialize, deserialize} pragmas and options are not supported for cbor assertNoPragma(T, deserialize, "deserialize pragma not supported") try: diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index bdfefb6..0b24e46 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -60,20 +60,20 @@ proc writeInitial[T: SomeInteger](str: Stream, m: uint8, n: T): ?!void = str.write(n.uint8) elif uint64(n) <= uint64(uint16.high): str.write(m or 25'u8) - str.write((uint8)n shr 8) - str.write((uint8)n) + str.write((uint8) n shr 8) + str.write((uint8) n) elif uint64(n) <= uint64(uint32.high): str.write(m or 26'u8) for i in countdown(24, 8, 8): {.unroll.} - str.write((uint8)n shr i) - str.write((uint8)n) + str.write((uint8) n shr i) + str.write((uint8) n) else: str.write(m or 27'u8) for i in countdown(56, 8, 8): {.unroll.} - str.write((uint8)n shr i) - str.write((uint8)n) + str.write((uint8) n shr i) + str.write((uint8) n) success() except IOError as e: return failure(e.msg) @@ -200,20 +200,20 @@ proc writeCbor*[T: SomeFloat](str: Stream, v: T): ?!void = return success() of fcZero: str.write initialByte(7, 25) - str.write((char)0x00) + str.write((char) 0x00) of fcNegZero: str.write initialByte(7, 25) - str.write((char)0x80) + str.write((char) 0x80) of fcInf: str.write initialByte(7, 25) - str.write((char)0x7c) + str.write((char) 0x7c) of fcNan: str.write initialByte(7, 25) - str.write((char)0x7e) + str.write((char) 0x7e) of fcNegInf: str.write initialByte(7, 25) - str.write((char)0xfc) - str.write((char)0x00) + str.write((char) 0xfc) + str.write((char) 0x00) success() except IOError as io: return failure(io.msg) @@ -267,7 +267,7 @@ proc writeCbor*(str: Stream, v: CborNode): ?!void = proc writeCbor*[T: object](str: Stream, v: T): ?!void = var n: uint - # Added because serde {serialize, deserialize} pragma and options are not supported cbor + # Added because serde {serialize, deserialize} pragma and options are not supported for cbor assertNoPragma(T, serialize, "serialize pragma not supported") for _, _ in v.fieldPairs: @@ -405,7 +405,7 @@ func initCborBytes*[T: char | byte](buf: openArray[T]): CborNode = ## Create a CBOR byte string from `buf`. result = CborNode(kind: cborBytes, bytes: newSeq[byte](buf.len)) for i in 0 ..< buf.len: - result.bytes[i] = (byte)buf[i] + result.bytes[i] = (byte) buf[i] func initCborBytes*(len: int): CborNode = ## Create a CBOR byte string of ``len`` bytes. diff --git a/serde/json/README.md b/serde/json/README.md new file mode 100644 index 0000000..9aa0af9 --- /dev/null +++ b/serde/json/README.md @@ -0,0 +1,421 @@ +# nim-serde JSON + +Explore JSON serialization and deserialization using nim-serde, an improved alternative to `std/json`. + +## Table of Contents +- [nim-serde JSON](#nim-serde-json) + - [Table of Contents](#table-of-contents) + - [Serialization API](#serialization-api) + - [Basic Serialization with `%` operator](#basic-serialization-with--operator) + - [Object Serialization](#object-serialization) + - [Serialization with `%*`](#serialization-with-) + - [Converting to JSON String with `toJson`](#converting-to-json-string-with-tojson) + - [Serialization Modes](#serialization-modes) + - [Field Customization for Serialization](#field-customization-for-serialization) + - [Deserialization API](#deserialization-api) + - [Basic Deserialization with `fromJson`](#basic-deserialization-with-fromjson) + - [Error Handling](#error-handling) + - [Parsing JSON with `JsonNode.parse`](#parsing-json-with-jsonnodeparse) + - [Deserialization Modes](#deserialization-modes) + - [Field Customization for Deserialization](#field-customization-for-deserialization) + - [Using as a Drop-in Replacement for std/json](#using-as-a-drop-in-replacement-for-stdjson) + - [Custom Type Serialization](#custom-type-serialization) + - [Implementation Details](#implementation-details) + +## Serialization API + +The nim-serde JSON serialization API provides several ways to convert Nim values to JSON. + +### Basic Serialization with `%` operator + +The `%` operator converts Nim values to `JsonNode` objects, which can then be converted to JSON strings: + +```nim +import pkg/serde/json + +# Basic types +assert %42 == newJInt(42) +assert %"hello" == newJString("hello") +assert %true == newJBool(true) + +# Arrays and sequences +let arr = newJArray() +arr.add(newJInt(1)) +arr.add(newJInt(2)) +arr.add(newJInt(3)) +assert $(%[1, 2, 3]) == $arr +``` + +### Object Serialization + +Objects can be serialized using the `%` operator, which automatically handles field serialization based on the object's configuration: + +```nim +import pkg/serde/json + +type Person = object + name {.serialize.}: string + age {.serialize.}: int + address: string # Not serialized by default in OptIn mode + +let person = Person(name: "John", age: 30, address: "123 Main St") +let jsonNode = %person +assert jsonNode.kind == JObject +assert jsonNode.len == 2 +assert jsonNode["name"].getStr == "John" +assert jsonNode["age"].getInt == 30 +assert "address" notin jsonNode +``` + +### Serialization with `%*` + +The `%*` macro provides a more convenient way to create JSON objects: + +```nim +import pkg/serde/json + +let jsonObj = %*{ + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"], + "address": { + "street": "123 Main St", + "city": "Anytown" + } +} + +assert jsonObj.kind == JObject +assert jsonObj["name"].getStr == "John" +assert jsonObj["age"].getInt == 30 +assert jsonObj["hobbies"].kind == JArray +assert jsonObj["hobbies"][0].getStr == "reading" +assert jsonObj["address"]["street"].getStr == "123 Main St" +``` + +### Converting to JSON String with `toJson` + +The `toJson` function converts any serializable value directly to a JSON string: + +```nim +import pkg/serde/json + +type Person = object + name {.serialize.}: string + age {.serialize.}: int + +let person = Person(name: "John", age: 30) +assert person.toJson == """{"name":"John","age":30}""" +``` + +### Serialization Modes + +nim-serde offers three modes to control which fields are serialized: + +```nim +import pkg/serde/json + +# OptIn mode (default): Only fields with {.serialize.} are included +type Person1 = object + name {.serialize.}: string + age {.serialize.}: int + address: string # Not serialized + +assert Person1(name: "John", age: 30, address: "123 Main St").toJson == """{"name":"John","age":30}""" + +# OptOut mode: All fields are included except those marked to ignore +type Person2 {.serialize.} = object + name: string + age: int + ssn {.serialize(ignore=true).}: string # Not serialized + +assert Person2(name: "John", age: 30, ssn: "123-45-6789").toJson == """{"name":"John","age":30}""" + +# Strict mode: All fields are included, and an error is raised if any fields are missing +type Person3 {.serialize(mode=Strict).} = object + name: string + age: int + +assert Person3(name: "John", age: 30).toJson == """{"name":"John","age":30}""" +``` + +### Field Customization for Serialization + +Fields can be customized with various options: + +```nim +import pkg/serde/json + +# Field customization for serialization +type Person {.serialize(mode = OptOut).} = object + firstName {.serialize(key = "first_name").}: string + lastName {.serialize(key = "last_name").}: string + age: int # Will be included because we're using OptOut mode + ssn {.serialize(ignore = true).}: string # Sensitive data not serialized + +let person = Person( + firstName: "John", + lastName: "Doe", + age: 30, + ssn: "123-45-6789" +) + +let jsonNode = %person +assert jsonNode.kind == JObject +assert jsonNode["first_name"].getStr == "John" +assert jsonNode["last_name"].getStr == "Doe" +assert jsonNode["age"].getInt == 30 +assert "ssn" notin jsonNode + +# Convert to JSON string +let jsonStr = toJson(person) +assert jsonStr == """{"first_name":"John","last_name":"Doe","age":30}""" +``` + +## Deserialization API + +nim-serde provides a type-safe way to convert JSON data back into Nim types. + +### Basic Deserialization with `fromJson` + +The `fromJson` function converts JSON strings or `JsonNode` objects to Nim types: + +```nim +import pkg/serde/json +import pkg/questionable/results + +type Person = object + name: string + age: int + +let jsonStr = """{"name":"John","age":30}""" +let result = Person.fromJson(jsonStr) + +# Using the ! operator from questionable to extract the value +assert !result == Person(name: "John", age: 30) +``` + +### Error Handling + +Deserialization returns a `Result` type from the `questionable` library, allowing for safe error handling: + +```nim +import pkg/serde/json +import pkg/questionable/results + +type Person = object + name: string + age: int + +let invalidJson = """{"name":"John","age":"thirty"}""" +let errorResult = Person.fromJson(invalidJson) +assert errorResult.isFailure +assert errorResult.error of UnexpectedKindError +``` + +### Parsing JSON with `JsonNode.parse` + +For parsing raw JSON without immediate deserialization: + +```nim +import pkg/serde/json +import pkg/questionable/results + +let jsonStr = """{"name":"John","age":30,"hobbies":["reading","coding"]}""" +let parseResult = JsonNode.parse(jsonStr) +assert parseResult.isSuccess +let jsonNode = !parseResult +assert jsonNode["name"].getStr == "John" +assert jsonNode["age"].getInt == 30 +assert jsonNode["hobbies"].kind == JArray +assert jsonNode["hobbies"][0].getStr == "reading" +assert jsonNode["hobbies"][1].getStr == "coding" +``` + +### Deserialization Modes + +nim-serde offers three modes to control how JSON is deserialized: + +```nim +import pkg/serde/json +import pkg/questionable/results + +# OptOut mode (default for deserialization) +type PersonOptOut = object + name: string + age: int + +let jsonOptOut = """{"name":"John","age":30,"address":"123 Main St"}""" +let resultOptOut = PersonOptOut.fromJson(jsonOptOut) +assert resultOptOut.isSuccess +assert !resultOptOut == PersonOptOut(name: "John", age: 30) + +# OptIn mode +type PersonOptIn {.deserialize(mode = OptIn).} = object + name {.deserialize.}: string + age {.deserialize.}: int + address: string # Not deserialized by default in OptIn mode + +let jsonOptIn = """{"name":"John","age":30,"address":"123 Main St"}""" +let resultOptIn = PersonOptIn.fromJson(jsonOptIn) +assert resultOptIn.isSuccess +assert (!resultOptIn).name == "John" +assert (!resultOptIn).age == 30 +assert (!resultOptIn).address == "" # address is not deserialized + +# Strict mode +type PersonStrict {.deserialize(mode = Strict).} = object + name: string + age: int + +let jsonStrict = """{"name":"John","age":30}""" +let resultStrict = PersonStrict.fromJson(jsonStrict) +assert resultStrict.isSuccess +assert !resultStrict == PersonStrict(name: "John", age: 30) + +# Strict mode with extra field (should fail) +let jsonStrictExtra = """{"name":"John","age":30,"address":"123 Main St"}""" +let resultStrictExtra = PersonStrict.fromJson(jsonStrictExtra) +assert resultStrictExtra.isFailure +``` + +### Field Customization for Deserialization + +Fields can be customized with various options for deserialization: + +```nim +import pkg/serde/json +import pkg/questionable/results + +type User = object + firstName {.deserialize(key = "first_name").}: string + lastName {.deserialize(key = "last_name").}: string + age: int + internalId {.deserialize(ignore = true).}: int + +let userJsonStr = """{"first_name":"Jane","last_name":"Smith","age":25,"role":"admin"}""" +let result = User.fromJson(userJsonStr) +assert result.isSuccess +assert (!result).firstName == "Jane" +assert (!result).lastName == "Smith" +assert (!result).age == 25 +assert (!result).internalId == 0 # Default value, not deserialized +``` + +## Using as a Drop-in Replacement for std/json + +nim-serde can be used as a drop-in replacement for the standard library's `json` module with improved exception handling. + +```nim +# Instead of: +# import std/json +import pkg/serde/json +import pkg/questionable/results + +# Using nim-serde's JSON API which is compatible with std/json +let jsonNode = %* { + "name": "John", + "age": 30, + "isActive": true, + "hobbies": ["reading", "swimming"] +} + +# Accessing JSON fields using the same API as std/json +assert jsonNode.kind == JObject +assert jsonNode["name"].getStr == "John" +assert jsonNode["age"].getInt == 30 +assert jsonNode["isActive"].getBool == true +assert jsonNode["hobbies"].kind == JArray +assert jsonNode["hobbies"][0].getStr == "reading" +assert jsonNode["hobbies"][1].getStr == "swimming" + +# Converting JSON to string +let jsonStr = $jsonNode + +# Parsing JSON from string with better error handling +let parsedResult = JsonNode.parse(jsonStr) +assert parsedResult.isSuccess +let parsedNode = !parsedResult +assert parsedNode.kind == JObject +assert parsedNode["name"].getStr == "John" + +# Pretty printing +let prettyJson = pretty(jsonNode) +``` + +## Custom Type Serialization + +You can extend nim-serde to support custom types by defining your own `%` operator overloads and `fromJson` procs: + +```nim +import pkg/serde/json +import pkg/serde/utils/errors +import pkg/questionable/results +import std/strutils + +# Define a custom type +type + UserId = distinct int + +# Custom serialization for UserId +proc `%`*(id: UserId): JsonNode = + %("user-" & $int(id)) + +# Custom deserialization for UserId +proc fromJson*(_: type UserId, json: JsonNode): ?!UserId = + if json.kind != JString: + return failure(newSerdeError("Expected string for UserId, got " & $json.kind)) + + let str = json.getStr() + if str.startsWith("user-"): + let idStr = str[5..^1] + try: + let id = parseInt(idStr) + success(UserId(id)) + except ValueError: + failure(newSerdeError("Invalid UserId format: " & str)) + else: + failure(newSerdeError("UserId must start with 'user-' prefix")) + +# Test serialization +let userId = UserId(42) +let jsonNode = %userId +assert jsonNode.kind == JString +assert jsonNode.getStr() == "user-42" + +# Test deserialization +let jsonStr = "\"user-42\"" +let parsedJson = !JsonNode.parse(jsonStr) +let result = UserId.fromJson(parsedJson) +assert result.isSuccess +assert int(!result) == 42 + +# Test in object context +type User {.serialize(mode = OptOut).} = object + id: UserId + name: string + +let user = User(id: UserId(123), name: "John") +let userJson = %user +assert userJson.kind == JObject +assert userJson["id"].getStr() == "user-123" +assert userJson["name"].getStr() == "John" + +# Test deserialization of object with custom type +let userJsonStr = """{"id":"user-123","name":"John"}""" +let userResult = User.fromJson(userJsonStr) +assert userResult.isSuccess +assert int((!userResult).id) == 123 +assert (!userResult).name == "John" +``` + +## Implementation Details + +The JSON serialization in nim-serde is based on the `%` operator pattern: + +1. The `%` operator converts values to `JsonNode` objects +2. Various overloads handle different types (primitives, objects, collections) +3. The `toJson` function converts the `JsonNode` to a string + +This approach makes it easy to extend with custom types by defining your own `%` operator overloads. + +For deserialization, the library uses compile-time reflection to map JSON fields to object fields, respecting the configuration provided by pragmas. From fc799d25de2ce8f4c70895069113d97c803d0983 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Mon, 9 Jun 2025 12:40:28 +0530 Subject: [PATCH 18/20] cleanup json and cbor readme --- serde/cbor/README.md | 23 ++-- serde/cbor/helpers.nim | 4 - serde/json/README.md | 260 +++++++++++++++++++++++++++++------------ 3 files changed, 198 insertions(+), 89 deletions(-) diff --git a/serde/cbor/README.md b/serde/cbor/README.md index f546c4d..1478de1 100644 --- a/serde/cbor/README.md +++ b/serde/cbor/README.md @@ -1,15 +1,18 @@ # nim-serde CBOR -This README details the usage of CBOR serialization and deserialization features offered by nim-serde, in compliance with [RFC 8949](https://datatracker.ietf.org/doc/html/rfc8949). +The CBOR module in nim-serde provides serialization and deserialization for Nim values following [RFC 8949](https://www.rfc-editor.org/rfc/rfc8949.html). Unlike the JSON implementation, CBOR offers a stream-based API that enables direct binary serialization without intermediate representations. + +> Note: While the JSON implementation supports serde modes via pragmas, the current CBOR implementation does not support the `serialize` and `deserialize` pragmas. The library will raise an assertion error if you try to use these pragmas with CBOR serialization. + ## Table of Contents - [nim-serde CBOR](#nim-serde-cbor) - [Table of Contents](#table-of-contents) - [Serialization API](#serialization-api) - - [Basic Serialization with Stream API](#basic-serialization-with-stream-api) - - [Object Serialization](#object-serialization) - - [Custom Type Serialization](#custom-type-serialization) - - [Converting to CBOR with `toCbor`](#converting-to-cbor-with-tocbor) + - [Stream API: Primitive Type and Sequence Serialization](#stream-api-primitive-type-and-sequence-serialization) + - [Stream API: Object Serialization](#stream-api-object-serialization) + - [Stream API: Custom Type Serialization](#stream-api-custom-type-serialization) + - [Serialization without Stream API: `toCbor`](#serialization-without-stream-api-tocbor) - [Working with CborNode](#working-with-cbornode) - [Convenience Functions for CborNode](#convenience-functions-for-cbornode) - [Deserialization API](#deserialization-api) @@ -24,7 +27,7 @@ This README details the usage of CBOR serialization and deserialization features The nim-serde CBOR serialization API provides several ways to convert Nim values to CBOR. -### Basic Serialization with Stream API +### Stream API: Primitive Type and Sequence Serialization The `writeCbor` function writes Nim values to a stream in CBOR format: @@ -50,7 +53,7 @@ discard stream.writeCbor(@[1, 2, 3]) # Sequence let cborData = stream.data ``` -### Object Serialization +### Stream API: Object Serialization Objects can be serialized to CBOR format using the stream API: @@ -78,7 +81,7 @@ discard stream.writeCbor(person) let cborData = stream.data ``` -### Custom Type Serialization +### Stream API: Custom Type Serialization You can extend nim-serde to support custom types by defining your own `writeCbor` procs: @@ -114,7 +117,7 @@ discard userStream.writeCbor(user) let userCborData = userStream.data ``` -### Converting to CBOR with `toCbor` +### Serialization without Stream API: `toCbor` The `toCbor` function can be used to directly convert a Nim value to CBOR binary data: @@ -363,7 +366,7 @@ For deserialization, the library parses CBOR data into a `CborNode` representati ### Current Limitations -While the JSON implementation supports serde modes via pragmas, the current CBOR implementation does not support the `serialize` and `deserialize` pragmas. The library will raise an assertion error if you try to use these pragmas with CBOR serialization. +This implementation does not support the `serialize` and `deserialize` pragmas. The library will raise an assertion error if you try to use these pragmas with CBOR serialization. ```nim import pkg/serde/cbor diff --git a/serde/cbor/helpers.nim b/serde/cbor/helpers.nim index dc26430..5dc6176 100644 --- a/serde/cbor/helpers.nim +++ b/serde/cbor/helpers.nim @@ -36,10 +36,6 @@ template assertNoPragma*(value, pragma, msg) = when value.hasCustomPragma(pragma): raiseAssert(msg) -macro dot*(obj: object, fld: string): untyped = - ## Turn ``obj.dot("fld")`` into ``obj.fld``. - newDotExpr(obj, newIdentNode(fld.strVal)) - func floatSingle*(half: uint16): float32 = ## Convert a 16-bit float to 32-bits. func ldexp( diff --git a/serde/json/README.md b/serde/json/README.md index 9aa0af9..cdc036c 100644 --- a/serde/json/README.md +++ b/serde/json/README.md @@ -1,17 +1,22 @@ # nim-serde JSON -Explore JSON serialization and deserialization using nim-serde, an improved alternative to `std/json`. +The JSON module in nim-serde provides serialization and deserialization for Nim values, offering an improved alternative to the standard `std/json` library. Unlike the standard library, nim-serde JSON implements a flexible system of serialization/deserialization modes that give developers precise control over how Nim objects are converted to and from JSON. ## Table of Contents - [nim-serde JSON](#nim-serde-json) - [Table of Contents](#table-of-contents) + - [Serde Modes](#serde-modes) + - [Modes Overview](#modes-overview) + - [Default Modes](#default-modes) + - [Field Options](#field-options) - [Serialization API](#serialization-api) - [Basic Serialization with `%` operator](#basic-serialization-with--operator) - [Object Serialization](#object-serialization) - - [Serialization with `%*`](#serialization-with-) + - [Inlining JSON Directly in Code with `%*`](#inlining-json-directly-in-code-with-) - [Converting to JSON String with `toJson`](#converting-to-json-string-with-tojson) - [Serialization Modes](#serialization-modes) - [Field Customization for Serialization](#field-customization-for-serialization) + - [Custom Type Serialization](#custom-type-serialization) - [Deserialization API](#deserialization-api) - [Basic Deserialization with `fromJson`](#basic-deserialization-with-fromjson) - [Error Handling](#error-handling) @@ -19,9 +24,111 @@ Explore JSON serialization and deserialization using nim-serde, an improved alte - [Deserialization Modes](#deserialization-modes) - [Field Customization for Deserialization](#field-customization-for-deserialization) - [Using as a Drop-in Replacement for std/json](#using-as-a-drop-in-replacement-for-stdjson) - - [Custom Type Serialization](#custom-type-serialization) - [Implementation Details](#implementation-details) + +## Serde Modes +This implementation supports three different modes to control de/serialization: + +```nim +OptIn +OptOut +Strict +``` + +Modes can be set in the `{.serialize.}` and/or `{.deserialize.}` pragmas on type +definitions. Each mode has a different meaning depending on if the type is being +serialized or deserialized. Modes can be set by setting `mode` in the `serialize` or +`deserialize` pragma annotation, eg: + +```nim +type MyType {.serialize(mode=Strict).} = object + field1: bool + field2: bool +``` + +### Modes Overview + +| Mode | Serialize | Deserialize | +|:-----|:----------|:------------| +| `OptOut` | All object fields will be serialized, except fields marked with `{.serialize(ignore=true).}`. | All JSON keys will be deserialized, except fields marked with `{.deserialize(ignore=true).}`. No error if extra JSON fields exist. | +| `OptIn` | Only fields marked with `{.serialize.}` will be serialized. Fields marked with `{.serialize(ignore=true).}` will not be serialized. | Only fields marked with `{.deserialize.}` will be deserialized. Fields marked with `{.deserialize(ignore=true).}` will not be deserialized. A `SerdeError` is raised if the field is missing in JSON. | +| `Strict` | All object fields will be serialized, regardless if the field is marked with `{.serialize(ignore=true).}`. | Object fields and JSON fields must match exactly, otherwise a `SerdeError` is raised. | + +### Default Modes + +Types can be serialized and deserialized even without explicit annotations, using default modes. Without any pragmas, types are serialized in OptIn mode and deserialized in OptOut mode. When types have pragmas but no specific mode is set, OptOut mode is used for both serialization and deserialization. + + +| Context | Serialize | Deserialize | +|:--------|:----------|:------------| +| Default (no pragma) | `OptIn` | `OptOut` | +| Default (pragma, but no mode) | `OptOut` | `OptOut` | + +```nim +# Type is not annotated +# A default mode of OptIn (for serialize) and OptOut (for deserialize) is assumed. +type MyObj1 = object + field1: bool + field2: bool + +# Type is annotated, but mode not specified +# A default mode of OptOut is assumed for both serialize and deserialize. +type MyObj2 {.serialize, deserialize.} = object + field1: bool + field2: bool +``` + +### Field Options + +Individual fields can be customized using the `{.serialize.}` and `{.deserialize.}` pragmas with additional options that control how each field is processed during serialization and deserialization + + +| | serialize | deserialize | +|:---------|:-----------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| +| `key` | aliases the field name in json | deserializes the field if json contains `key` | +| `ignore` |
  • **OptOut:** field not serialized
  • **OptIn:** field not serialized
  • **Strict:** field serialized
    • |
  • **OptOut:** field not deserialized
  • **OptIn:** field not deserialized
  • **Strict:** field deserialized
    • | + + +Example with field options: + +```nim +import pkg/serde/json + +type + Person {.serialize(mode=OptOut), deserialize(mode=OptIn).} = object + id {.serialize(ignore=true), deserialize(key="personid").}: int + name: string + birthYear: int + address: string + phone: string + +let person = Person( + name: "Lloyd Christmas", + birthYear: 1970, + address: "123 Sesame Street, Providence, Rhode Island 12345", + phone: "555-905-justgivemethedamnnumber!⛽️🔥") + +let createRequest = """{ + "name": "Lloyd Christmas", + "birthYear": 1970, + "address": "123 Sesame Street, Providence, Rhode Island 12345", + "phone": "555-905-justgivemethedamnnumber!⛽️🔥" +}""" +assert person.toJson(pretty=true) == createRequest + +let createResponse = """{ + "personid": 1, + "name": "Lloyd Christmas", + "birthYear": 1970, + "address": "123 Sesame Street, Providence, Rhode Island 12345", + "phone": "555-905-justgivemethedamnnumber!⛽️🔥" +}""" +assert !Person.fromJson(createResponse) == Person(id: 1) +``` + +More examples can be found in [Serialization Modes](#serialization-modes) and [Deserialization Modes](#deserialization-modes). + ## Serialization API The nim-serde JSON serialization API provides several ways to convert Nim values to JSON. @@ -67,16 +174,19 @@ assert jsonNode["age"].getInt == 30 assert "address" notin jsonNode ``` -### Serialization with `%*` +### Inlining JSON Directly in Code with `%*` The `%*` macro provides a more convenient way to create JSON objects: ```nim import pkg/serde/json -let jsonObj = %*{ - "name": "John", - "age": 30, +let + name = "John" + age = 30 + jsonObj = %*{ + "name": name, + "age": age, "hobbies": ["reading", "coding"], "address": { "street": "123 Main St", @@ -85,8 +195,8 @@ let jsonObj = %*{ } assert jsonObj.kind == JObject -assert jsonObj["name"].getStr == "John" -assert jsonObj["age"].getInt == 30 +assert jsonObj["name"].getStr == name +assert jsonObj["age"].getInt == age assert jsonObj["hobbies"].kind == JArray assert jsonObj["hobbies"][0].getStr == "reading" assert jsonObj["address"]["street"].getStr == "123 Main St" @@ -171,6 +281,72 @@ let jsonStr = toJson(person) assert jsonStr == """{"first_name":"John","last_name":"Doe","age":30}""" ``` +## Custom Type Serialization + +You can extend nim-serde to support custom types by defining your own `%` operator overloads and `fromJson` procs: + +```nim +import pkg/serde/json +import pkg/serde/utils/errors +import pkg/questionable/results +import std/strutils + +# Define a custom type +type + UserId = distinct int + +# Custom serialization for UserId +proc `%`*(id: UserId): JsonNode = + %("user-" & $int(id)) + +# Custom deserialization for UserId +proc fromJson*(_: type UserId, json: JsonNode): ?!UserId = + if json.kind != JString: + return failure(newSerdeError("Expected string for UserId, got " & $json.kind)) + + let str = json.getStr() + if str.startsWith("user-"): + let idStr = str[5..^1] + try: + let id = parseInt(idStr) + success(UserId(id)) + except ValueError: + failure(newSerdeError("Invalid UserId format: " & str)) + else: + failure(newSerdeError("UserId must start with 'user-' prefix")) + +# Test serialization +let userId = UserId(42) +let jsonNode = %userId +assert jsonNode.kind == JString +assert jsonNode.getStr() == "user-42" + +# Test deserialization +let jsonStr = "\"user-42\"" +let parsedJson = !JsonNode.parse(jsonStr) +let result = UserId.fromJson(parsedJson) +assert result.isSuccess +assert int(!result) == 42 + +# Test in object context +type User {.serialize(mode = OptOut).} = object + id: UserId + name: string + +let user = User(id: UserId(123), name: "John") +let userJson = %user +assert userJson.kind == JObject +assert userJson["id"].getStr() == "user-123" +assert userJson["name"].getStr() == "John" + +# Test deserialization of object with custom type +let userJsonStr = """{"id":"user-123","name":"John"}""" +let userResult = User.fromJson(userJsonStr) +assert userResult.isSuccess +assert int((!userResult).id) == 123 +assert (!userResult).name == "John" +``` + ## Deserialization API nim-serde provides a type-safe way to convert JSON data back into Nim types. @@ -342,72 +518,6 @@ assert parsedNode["name"].getStr == "John" let prettyJson = pretty(jsonNode) ``` -## Custom Type Serialization - -You can extend nim-serde to support custom types by defining your own `%` operator overloads and `fromJson` procs: - -```nim -import pkg/serde/json -import pkg/serde/utils/errors -import pkg/questionable/results -import std/strutils - -# Define a custom type -type - UserId = distinct int - -# Custom serialization for UserId -proc `%`*(id: UserId): JsonNode = - %("user-" & $int(id)) - -# Custom deserialization for UserId -proc fromJson*(_: type UserId, json: JsonNode): ?!UserId = - if json.kind != JString: - return failure(newSerdeError("Expected string for UserId, got " & $json.kind)) - - let str = json.getStr() - if str.startsWith("user-"): - let idStr = str[5..^1] - try: - let id = parseInt(idStr) - success(UserId(id)) - except ValueError: - failure(newSerdeError("Invalid UserId format: " & str)) - else: - failure(newSerdeError("UserId must start with 'user-' prefix")) - -# Test serialization -let userId = UserId(42) -let jsonNode = %userId -assert jsonNode.kind == JString -assert jsonNode.getStr() == "user-42" - -# Test deserialization -let jsonStr = "\"user-42\"" -let parsedJson = !JsonNode.parse(jsonStr) -let result = UserId.fromJson(parsedJson) -assert result.isSuccess -assert int(!result) == 42 - -# Test in object context -type User {.serialize(mode = OptOut).} = object - id: UserId - name: string - -let user = User(id: UserId(123), name: "John") -let userJson = %user -assert userJson.kind == JObject -assert userJson["id"].getStr() == "user-123" -assert userJson["name"].getStr() == "John" - -# Test deserialization of object with custom type -let userJsonStr = """{"id":"user-123","name":"John"}""" -let userResult = User.fromJson(userJsonStr) -assert userResult.isSuccess -assert int((!userResult).id) == 123 -assert (!userResult).name == "John" -``` - ## Implementation Details The JSON serialization in nim-serde is based on the `%` operator pattern: From 6793ee53f4f3929dda4bcc09f882efb0f5424d84 Mon Sep 17 00:00:00 2001 From: munna0908 Date: Mon, 9 Jun 2025 13:28:28 +0530 Subject: [PATCH 19/20] update readme with examples --- README.md | 194 +++++++++++++++++++++++++----------------------------- 1 file changed, 90 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 1660ecc..197afe6 100644 --- a/README.md +++ b/README.md @@ -1,132 +1,118 @@ # nim-serde -A serialization and deserialization library for Nim supporting multiple wire formats. +A serialization and deserialization library for Nim supporting multiple formats. -## Supported Wire Formats +## Supported Serialization Formats -nim-serde currently supports the following wire formats: +nim-serde currently supports the following serialization formats: - **JSON**: A text-based data interchange format. [See JSON](serde/json/README.md) for details. - **CBOR**: A binary data format following RFC 8949. [See CBOR](serde/cbor/README.md) for details. -## Serde modes +## Quick Examples -`nim-serde` uses three different modes to control de/serialization: +### JSON Serialization and Deserialization ```nim -OptIn -OptOut -Strict -``` +import ./serde/json +import questionable/results -Modes can be set in the `{.serialize.}` and/or `{.deserialize.}` pragmas on type -definitions. Each mode has a different meaning depending on if the type is being -serialized or deserialized. Modes can be set by setting `mode` in the `serialize` or -`deserialize` pragma annotation, eg: +# Define a type +type Person = object + name {.serialize.}: string + age {.serialize.}: int + address: string # Not serialized by default in OptIn mode -```nim -type MyType {.serialize(mode=Strict).} = object - field1: bool - field2: bool -``` +# Create an instance +let person = Person(name: "John Doe", age: 30, address: "123 Main St") -### Modes reference +# Serialization +echo "JSON Serialization Example" +let jsonString = person.toJson(pretty = true) +echo jsonString -| | serialize | deserialize | -|:-------------------|:------------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `SerdeMode.OptOut` | All object fields will be serialized, except fields marked with `{.serialize(ignore=true).}`. | All json keys will be deserialized, except fields marked with `{.deserialize(ignore=true).}`. No error if extra json fields exist. | -| `SerdeMode.OptIn` | Only fields marked with `{.serialize.}` will be serialized. Fields marked with `{.serialize(ignore=true).}` will not be serialized. | Only fields marked with `{.deserialize.}` will be deserialized. Fields marked with `{.deserialize(ignore=true).}` will not be deserialized. A `SerdeError` error is raised if the field is missing in json. | -| `SerdeMode.Strict` | All object fields will be serialized, regardless if the field is marked with `{.serialize(ignore=true).}`. | Object fields and json fields must match exactly, otherwise a `SerdeError` is raised. | - -## Default modes - -`nim-serde` will de/serialize types if they are not annotated with `serialize` or -`deserialize`, but will assume a default mode. By default, with no pragmas specified, -`serde` will always serialize in `OptIn` mode, meaning any fields to b Additionally, if -the types are annotated, but a mode is not specified, `serde` will assume a (possibly -different) default mode. - -```nim -# Type is not annotated -# A default mode of OptIn (for serialize) and OptOut (for deserialize) is assumed. - -type MyObj = object - field1: bool - field2: bool - -# Type is annotated, but mode not specified -# A default mode of OptOut is assumed for both serialize and deserialize. - -type MyObj {.serialize, deserialize.} = object - field1: bool - field2: bool -``` - -### Default mode reference - -| | serialize | deserialize | -|:------------------------------|:----------|:------------| -| Default (no pragma) | `OptIn` | `OptOut` | -| Default (pragma, but no mode) | `OptOut` | `OptOut` | - -## Serde field options -Type fields can be annotated with `{.serialize.}` and `{.deserialize.}` and properties -can be set on these pragmas, determining de/serialization behavior. - -For example, - -```nim -import pkg/serde/json - -type - Person {.serialize(mode=OptOut), deserialize(mode=OptIn).} = object - id {.serialize(ignore=true), deserialize(key="personid").}: int - name: string - birthYear: int - address: string - phone: string - -let person = Person( - name: "Lloyd Christmas", - birthYear: 1970, - address: "123 Sesame Street, Providence, Rhode Island 12345", - phone: "555-905-justgivemethedamnnumber!⛽️🔥") - -let createRequest = """{ - "name": "Lloyd Christmas", - "birthYear": 1970, - "address": "123 Sesame Street, Providence, Rhode Island 12345", - "phone": "555-905-justgivemethedamnnumber!⛽️🔥" +# Verify serialization output +let expectedJson = """{ + "name": "John Doe", + "age": 30 }""" -assert person.toJson(pretty=true) == createRequest +assert jsonString == expectedJson + +# Deserialization +echo "\nJSON Deserialization Example" +let jsonData = """{"name":"Jane Doe","age":28,"address":"456 Oak Ave"}""" +let result = Person.fromJson(jsonData) + +# check if deserialization was successful +assert result.isSuccess + +# get the deserialized value +let parsedPerson = !result + +echo parsedPerson +#[ +Expected Output: +Person( + name: "Jane Doe", + age: 28, + address: "456 Oak Ave" +) +]# -let createResponse = """{ - "personid": 1, - "name": "Lloyd Christmas", - "birthYear": 1970, - "address": "123 Sesame Street, Providence, Rhode Island 12345", - "phone": "555-905-justgivemethedamnnumber!⛽️🔥" -}""" -assert !Person.fromJson(createResponse) == Person(id: 1) ``` -### `key` -Specifying a `key`, will alias the field name. When seriazlizing, json will be written -with `key` instead of the field name. When deserializing, the json must contain `key` -for the field to be deserialized. +### CBOR Serialization and Deserialization -### `ignore` -Specifying `ignore`, will prevent de/serialization on the field. +```nim +import ./serde/cbor +import questionable/results +import std/streams -### Serde field options reference +# Define a type +type Person = object + name: string + age: int + address: string -| | serialize | deserialize | -|:---------|:-----------------------------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| -| `key` | aliases the field name in json | deserializes the field if json contains `key` | -| `ignore` |
  • **OptOut:** field not serialized
  • **OptIn:** field not serialized
  • **Strict:** field serialized
    • |
  • **OptOut:** field not deserialized
  • **OptIn:** field not deserialized
  • **Strict:** field deserialized
    • | +# Create an instance +let person = Person(name: "John Doe", age: 30, address: "123 Main St") +# Serialization using Stream API +echo "CBOR Stream API Serialization" +let stream = newStringStream() +let writeResult = stream.writeCbor(person) +assert writeResult.isSuccess +# Serialization using toCbor function +echo "\nCBOR toCbor Function Serialization" +let cborResult = toCbor(person) +assert cborResult.isSuccess +let serializedCbor = !cborResult + +# Deserialization +echo "\nCBOR Deserialization" +let personResult = Person.fromCbor(serializedCbor) + +# check if deserialization was successful +assert personResult.isSuccess + +# get the deserialized value +let parsedPerson = !personResult +echo parsedPerson + +#[ +Expected Output: +Person( + name: "John Doe", + age: 30, + address: "123 Main St" +) +]# + +``` + +Refer to the [json](serde/json/README.md) and [cbor](serde/cbor/README.md) files for more comprehensive examples. ## Known Issues From 2d8fa4d940a19a59551a59d79ca4dc38ce6ef104 Mon Sep 17 00:00:00 2001 From: munna0908 <88337208+munna0908@users.noreply.github.com> Date: Tue, 17 Jun 2025 12:37:17 +0530 Subject: [PATCH 20/20] Apply suggestions from code review Co-authored-by: Giuliano Mega --- README.md | 4 ++-- serde/json/README.md | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 197afe6..8fb7f2e 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ import questionable/results type Person = object name {.serialize.}: string age {.serialize.}: int - address: string # Not serialized by default in OptIn mode + address: string # By default, serde will not serialize non-annotated fields (OptIn mode) # Create an instance let person = Person(name: "John Doe", age: 30, address: "123 Main St") @@ -70,7 +70,7 @@ import std/streams # Define a type type Person = object - name: string + name: string # Unlike JSON, CBOR always serializes all fields, and they do not need to be annotated age: int address: string diff --git a/serde/json/README.md b/serde/json/README.md index cdc036c..ea2ae34 100644 --- a/serde/json/README.md +++ b/serde/json/README.md @@ -67,12 +67,14 @@ Types can be serialized and deserialized even without explicit annotations, usin ```nim # Type is not annotated -# A default mode of OptIn (for serialize) and OptOut (for deserialize) is assumed. +# If you don't annotate the type, serde assumes OptIn by default for serialization, and OptOut for +# deserialization. This means your types will be serialized to an empty string, which is probably not what you want: type MyObj1 = object field1: bool field2: bool -# Type is annotated, but mode not specified +# If you annotate your type but do not specify the mode, serde will default to OptOut for +# both serialize and de-serialize, meaning all fields get serialized/de-serialized by default: # A default mode of OptOut is assumed for both serialize and deserialize. type MyObj2 {.serialize, deserialize.} = object field1: bool @@ -390,7 +392,7 @@ assert errorResult.error of UnexpectedKindError ### Parsing JSON with `JsonNode.parse` -For parsing raw JSON without immediate deserialization: +To parse JSON string into a `JsonNode` tree instead of a deserializing to a concrete type, use `JsonNode.parse`: ```nim import pkg/serde/json