diff --git a/README.md b/README.md index 1b0d338..1660ecc 100644 --- a/README.md +++ b/README.md @@ -1,102 +1,13 @@ # nim-serde -Easy-to-use json serialization capabilities, and a drop-in replacement for `std/json`. +A serialization and deserialization library for Nim supporting multiple wire formats. -## Quick examples +## Supported Wire Formats -Opt-in serialization by default: +nim-serde currently supports the following wire formats: -```nim -import pkg/serde/json - -type MyType = object - field1 {.serialize.}: bool - field2: bool - -assert MyType(field1: true, field2: true).toJson == """{"field1":true}""" -``` - -Opt-out deserialization by default: - -```nim -import pkg/serde/json - -# All fields deserialized, as none are ignored -type MyType1 = object - field1: bool - field2: bool - -let jsn1 = """{ - "field1": true, - "field2": true - }""" -assert !MyType1.fromJson(jsn1) == MyType1(field1: true, field2: true) - -# Don't deserialize ignored fields in OptOut mode -type MyType2 = object - field1 {.deserialize(ignore=true).}: bool - field2: bool - -let jsn2 = """{ - "field1": true, - "field2": true, - "extra": "extra fields don't error in OptOut mode" - }""" -assert !MyType2.fromJson(jsn2) == MyType2(field1: false, field2: true) - -# Note, the ! operator is part of https://github.com/codex-storage/questionable, which retrieves a value if set -``` - -Serialize all fields of a type (OptOut mode): - -```nim -import pkg/serde/json - -type MyType {.serialize.} = object - field1: int - field2: int - -assert MyType(field1: 1, field2: 2).toJson == """{"field1":1,"field2":2}""" -``` - -Alias field names in both directions! - -```nim -import pkg/serde/json - -type MyType {.serialize.} = object - field1 {.serialize("othername"),deserialize("takesprecedence").}: int - field2: int - -assert MyType(field1: 1, field2: 2).toJson == """{"othername":1,"field2":2}""" -let jsn = """{ - "othername": 1, - "field2": 2, - "takesprecedence": 3 - }""" -assert !MyType.fromJson(jsn) == MyType(field1: 3, field2: 2) -``` - -Supports strict mode, where type fields and json fields must match - -```nim -import pkg/serde/json - -type MyType {.deserialize(mode=Strict).} = object - field1: int - field2: int - -let jsn = """{ - "field1": 1, - "field2": 2, - "extra": 3 - }""" - -let res = MyType.fromJson(jsn) -assert res.isFailure -assert res.error of SerdeError -assert res.error.msg == "json field(s) missing in object: {\"extra\"}" -``` +- **JSON**: A text-based data interchange format. [See JSON](serde/json/README.md) for details. +- **CBOR**: A binary data format following RFC 8949. [See CBOR](serde/cbor/README.md) for details. ## Serde modes @@ -215,198 +126,20 @@ Specifying `ignore`, will prevent de/serialization on the field. | `ignore` |
  • **OptOut:** field not serialized
  • **OptIn:** field not serialized
  • **Strict:** field serialized
  • |
  • **OptOut:** field not deserialized
  • **OptIn:** field not deserialized
  • **Strict:** field deserialized
  • | -## Deserialization -`serde` deserializes using `fromJson`, and in all instances returns `Result[T, -CatchableError]`, where `T` is the type being deserialized. For example: -```nim -type MyType = object - field1: bool - field2: bool +## Known Issues -let jsn1 = """{ - "field1": true, - "field2": true - }""" +There is a known issue when using mixins with generic overloaded procs like `fromJson`. At the time of mixin call, only the `fromJson` overloads in scope of the called mixin are available to be dispatched at runtime. There could be other `fromJson` overloads declared in other modules, but are not in scope at the time the mixin was called. -assert !MyType.fromJson(jsn1) == MyType(field1: true, field2: true) -``` +Therefore, anytime `fromJson` is called targeting a declared overload, it may or may not be dispatchable. This can be worked around by forcing the `fromJson` overload into scope at compile time. For example, in your application where the `fromJson` overload is defined, at the bottom of the module add: -If there was an error during deserialization, the result of `fromJson` will contain it: - -```nim -import pkg/serde/json - -type MyType {.deserialize(mode=Strict).} = object - field1: int - field2: int - -let jsn = """{ - "field1": 1, - "field2": 2, - "extra": 3 - }""" - -let res = MyType.fromJson(jsn) -assert res.isFailure -assert res.error of SerdeError -assert res.error.msg == "json field(s) missing in object: {\"extra\"}" -``` - -## Custom types - -If `serde` can't de/serialize a custom type, de/serialization can be supported by -overloading `%` and `fromJson`. For example: - -```nim -type - Address* = distinct array[20, byte] - SerializationError* = object of CatchableError - -func `%`*(address: Address): JsonNode = - %($address) - -func fromJson(_: type Address, json: JsonNode): ?!Address = - expectJsonKind(Address, JString, json) - without address =? Address.init(json.getStr), error: - return failure newException(SerializationError, - "Failed to convert '" & $json & "' to Address: " & error.msg) - success address -``` - -## Serializing to string (`toJson`) - -`toJson` is a shortcut for serializing an object into its serialized string -representation: - -```nim -import pkg/serde/json - -type MyType {.serialize.} = object - field1: string - field2: bool - -let mt = MyType(field1: "hw", field2: true) -assert mt.toJson == """{"field1":"hw","field2":true}""" -``` - -This comes in handy, for example, when sending API responses: - -```nim -let availability = getAvailability(...) -return RestApiResponse.response(availability.toJson, - contentType="application/json") -``` - -## `std/json` drop-in replacment - -`nim-serde` can be used as a drop-in replacement for the [standard library's `json` -module](https://nim-lang.org/docs/json.html), with a few notable improvements. - -Instead of importing `std/json` into your application, `pkg/serde/json` can be imported -instead: - -```diff -- import std/json -+ import pkg/serde/json -``` - -As with `std/json`, `%` can be used to serialize a type into a `JsonNode`: - -```nim -import pkg/serde/json - -assert %"hello" == newJString("hello") -``` - -And `%*` can be used to serialize objects: - -```nim -import pkg/serde/json - -let expected = newJObject() -expected["hello"] = newJString("world") -assert %*{"hello": "world"} == expected -``` - -As well, serialization of types can be overridden, and serialization of custom types can -be introduced. Here, we are overriding the serialization of `int`: - -```nim -import pkg/serde/json - -func `%`(i: int): JsonNode = - newJInt(i + 1) - -assert 1.toJson == "2" -``` - -## `parseJson` and exception tracking - -Unfortunately, `std/json`'s `parseJson` can raise an `Exception`, so proper exception -tracking breaks, eg - -```nim - -## Fails to compile: -## Error: parseJson(me, false, false) can raise an unlisted exception: Exception - -import std/json - -{.push raises:[].} - -type - MyAppError = object of CatchableError - -proc parseMe(me: string): JsonNode = - try: - return me.parseJson - except CatchableError as error: - raise newException(MyAppError, error.msg) - -assert """{"hello":"world"}""".parseMe == %* { "hello": "world" } -``` - -This is due to `std/json`'s `parseJson` incorrectly raising `Exception`. This can be -worked around by instead importing `serde` and calling its `JsonNode.parse` routine. -Note that `serde`'s `JsonNode.parse` returns a `Result[JsonNode, CatchableError]` -instead of just a plain `JsonNode` object as in `std/json`'s `parseJson`: - -```nim -import pkg/serde/json - -{.push raises:[].} - -type - MyAppError = object of CatchableError - -proc parseMe(me: string): JsonNode {.raises: [MyAppError].} = - without parsed =? JsonNode.parse(me), error: - raise newException(MyAppError, error.msg) - parsed - -assert """{"hello":"world"}""".parseMe == %* { "hello": "world" } -``` - -## Known issues - -There is a known issue when using mixins with generic overloaded procs like -`fromJson`. At the time of mixin call, only the `fromJson` overloads in scope of -the called mixin are available to be dispatched at runtime. There could be other -`fromJson` overloads declared in other modules, but are not in scope at the time -the mixin was called. Therefore, anytime `fromJson` is called targeting a -declared overload, it may or may not be dispatchable. This can be worked around -by forcing the `fromJson` overload into scope at compile time. For example, in -your application where the `fromJson` overload is defined, at the bottom of the -module add: ```nim static: MyType.fromJson("") ``` + This will ensure that the `MyType.fromJson` overload is dispatchable. -The basic types that serde supports should already have their overloads forced -in scope in [the `deserializer` -module](./serde/json/deserializer.nim#L340-L356). +The basic types that serde supports should already have their overloads forced in scope in [the `deserializer` module](./serde/json/deserializer.nim#L340-L356). For an illustration of the problem, please see this [narrow example](https://github.com/gmega/serialization-bug/tree/main/narrow) by [@gmega](https://github.com/gmega). diff --git a/serde/cbor/README.md b/serde/cbor/README.md new file mode 100644 index 0000000..f546c4d --- /dev/null +++ b/serde/cbor/README.md @@ -0,0 +1,376 @@ +# nim-serde CBOR + +This README details the usage of CBOR serialization and deserialization features offered by nim-serde, in compliance with [RFC 8949](https://datatracker.ietf.org/doc/html/rfc8949). + +## Table of Contents +- [nim-serde CBOR](#nim-serde-cbor) + - [Table of Contents](#table-of-contents) + - [Serialization API](#serialization-api) + - [Basic Serialization with Stream API](#basic-serialization-with-stream-api) + - [Object Serialization](#object-serialization) + - [Custom Type Serialization](#custom-type-serialization) + - [Converting to CBOR with `toCbor`](#converting-to-cbor-with-tocbor) + - [Working with CborNode](#working-with-cbornode) + - [Convenience Functions for CborNode](#convenience-functions-for-cbornode) + - [Deserialization API](#deserialization-api) + - [Basic Deserialization with `fromCbor`](#basic-deserialization-with-fromcbor) + - [Error Handling](#error-handling) + - [Parsing CBOR with `parseCbor`](#parsing-cbor-with-parsecbor) + - [Custom Type Deserialization](#custom-type-deserialization) + - [Implementation Details](#implementation-details) + - [Current Limitations](#current-limitations) + +## Serialization API + +The nim-serde CBOR serialization API provides several ways to convert Nim values to CBOR. + +### Basic Serialization with Stream API + +The `writeCbor` function writes Nim values to a stream in CBOR format: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/streams + +# Create a stream to write to +let stream = newStringStream() + +# Basic types +discard stream.writeCbor(42) # Unsigned integer +discard stream.writeCbor(-10) # Negative integer +discard stream.writeCbor(3.14) # Float +discard stream.writeCbor("hello") # String +discard stream.writeCbor(true) # Boolean + +# Arrays and sequences +discard stream.writeCbor(@[1, 2, 3]) # Sequence + +# Get the serialized CBOR data +let cborData = stream.data +``` + +### Object Serialization + +Objects can be serialized to CBOR format using the stream API: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/streams + +type Person = object + name: string + age: int + isActive: bool + +let person = Person( + name: "John", + age: 30, + isActive: true +) + +# Serialize the object to CBOR +let stream = newStringStream() +discard stream.writeCbor(person) + +# Get the serialized CBOR data +let cborData = stream.data +``` + +### Custom Type Serialization + +You can extend nim-serde to support custom types by defining your own `writeCbor` procs: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/streams +import std/strutils + +# Define a custom type +type + UserId = distinct int + +# Custom serialization for UserId +proc writeCbor*(str: Stream, id: UserId): ?!void = + # Write as a CBOR text string with a prefix + str.writeCbor("user-" & $int(id)) + +# Test serialization +let userId = UserId(42) +let stream = newStringStream() +discard stream.writeCbor(userId) +let cborData = stream.data + +# Test in object context +type User = object + id: UserId + name: string + +let user = User(id: UserId(123), name: "John") +let userStream = newStringStream() +discard userStream.writeCbor(user) +let userCborData = userStream.data +``` + +### Converting to CBOR with `toCbor` + +The `toCbor` function can be used to directly convert a Nim value to CBOR binary data: + +```nim +import pkg/serde/cbor +import pkg/questionable/results + +type Person = object + name: string + age: int + isActive: bool + +let person = Person( + name: "John", + age: 30, + isActive: true +) + +# Convert to CBOR binary data +let result = toCbor(person) +assert result.isSuccess +let cborData = !result +``` + +### Working with CborNode + +The `CborNode` type represents CBOR data in memory and can be manipulated directly: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/tables + +# Create CBOR nodes +let textNode = CborNode(kind: cborText, text: "hello") +let intNode = CborNode(kind: cborUnsigned, uint: 42'u64) +let floatNode = CborNode(kind: cborFloat, float: 3.14) + +# Create an array +var arrayNode = CborNode(kind: cborArray) +arrayNode.seq = @[textNode, intNode, floatNode] + +# Create a map with text keys and boolean values +var mapNode = CborNode(kind: cborMap) +mapNode.map = initOrderedTable[CborNode, CborNode]() +# Boolean values are represented as simple values (21 for true, 20 for false) +mapNode.map[CborNode(kind: cborText, text: "a")] = CborNode(kind: cborSimple, simple: 21) # true +mapNode.map[CborNode(kind: cborText, text: "b")] = CborNode(kind: cborSimple, simple: 20) # false + +# Convert to CBOR binary data +let result = toCbor(mapNode) +assert result.isSuccess +let cborData = !result +``` + +### Convenience Functions for CborNode + +The library provides convenience functions for creating CBOR nodes: + +```nim +import pkg/serde/cbor +import pkg/questionable/results + +# Initialize CBOR nodes +let bytesNode = initCborBytes(@[byte 1, byte 2, byte 3]) +let textNode = initCborText("hello") +let arrayNode = initCborArray() +let mapNode = initCborMap() + +# Convert values to CborNode +let intNodeResult = toCborNode(42) +assert intNodeResult.isSuccess +let intNode = !intNodeResult + +let strNodeResult = toCborNode("hello") +assert strNodeResult.isSuccess +let strNode = !strNodeResult + +let boolNodeResult = toCborNode(true) +assert boolNodeResult.isSuccess +let boolNode = !boolNodeResult +``` + +## Deserialization API + +The nim-serde CBOR deserialization API provides ways to convert CBOR data back to Nim values. + +### Basic Deserialization with `fromCbor` + +The `fromCbor` function converts CBOR data to Nim values: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/streams + +# Create some CBOR data +let stream = newStringStream() +discard stream.writeCbor(42) +let cborData = stream.data + +# Parse the CBOR data into a CborNode +try: + let node = parseCbor(cborData) + + # Deserialize the CborNode to a Nim value + let intResult = int.fromCbor(node) + assert intResult.isSuccess + let value = !intResult + assert value == 42 + + # You can also deserialize to other types + # For example, if cborData contained a string: + # let strResult = string.fromCbor(node) + # assert strResult.isSuccess + # let strValue = !strResult + +# Deserialize to an object +type Person = object + name: string + age: int + isActive: bool + +let personResult = Person.fromCbor(node) +assert personResult.isSuccess +let person = !personResult + +# Verify the deserialized data +assert person.name == "John" +assert person.age == 30 +assert person.isActive == true +``` + +### Error Handling + +Deserialization returns a `Result` type from the `questionable` library, allowing for safe error handling: + +```nim +import pkg/serde/cbor +import pkg/questionable/results + +# Invalid CBOR data for an integer +let invalidNode = CborNode(kind: cborText, text: "not an int") +let result = int.fromCbor(invalidNode) + +# Check for failure +assert result.isFailure +echo result.error.msg +# Output: "deserialization to int failed: expected {cborUnsigned, cborNegative} but got cborText" +``` + +### Parsing CBOR with `parseCbor` + +The `parseCbor` function parses CBOR binary data into a `CborNode`: + +```nim +import pkg/serde/cbor +import pkg/questionable/results + +# Parse CBOR data +let node = parseCbor(cborData) + +# Check node type and access data +case node.kind +of cborUnsigned: + echo "Unsigned integer: ", node.uint +of cborNegative: + echo "Negative integer: ", node.int +of cborText: + echo "Text: ", node.text +of cborArray: + echo "Array with ", node.seq.len, " items" +of cborMap: + echo "Map with ", node.map.len, " pairs" +else: + echo "Other CBOR type: ", node.kind +``` + +### Custom Type Deserialization + +You can extend nim-serde to support custom type deserialization by defining your own `fromCbor` procs: + +```nim +import pkg/serde/cbor +import pkg/questionable/results +import std/strutils + +# Define a custom type +type + UserId = distinct int + +# Custom deserialization for UserId +proc fromCbor*(_: type UserId, n: CborNode): ?!UserId = + if n.kind != cborText: + return failure(newSerdeError("Expected string for UserId, got " & $n.kind)) + + let str = n.text + if str.startsWith("user-"): + let idStr = str[5..^1] + try: + let id = parseInt(idStr) + success(UserId(id)) + except ValueError: + failure(newSerdeError("Invalid UserId format: " & str)) + else: + failure(newSerdeError("UserId must start with 'user-' prefix")) + +# Test deserialization +let node = parseCbor(cborData) # Assuming cborData contains a serialized UserId +let result = UserId.fromCbor(node) +assert result.isSuccess +assert int(!result) == 42 + +# Test deserialization in object context +type User = object + id: UserId + name: string + +let userNode = parseCbor(userCborData) # Assuming userCborData contains a serialized User +let userResult = User.fromCbor(userNode) +assert userResult.isSuccess +assert int((!userResult).id) == 123 +assert (!userResult).name == "John" +``` + + +## Implementation Details + +The CBOR serialization in nim-serde follows a stream-based approach: + +``` +# Serialization flow +Nim value → writeCbor → CBOR binary data + +# Deserialization flow +CBOR binary data → parseCbor (CborNode) → fromCbor → Nim value +``` + +Unlike the JSON implementation which uses the `%` operator pattern, the CBOR implementation uses a hook-based approach: + +1. The `writeCbor` function writes Nim values directly to a stream in CBOR format +2. Custom types can be supported by defining `writeCbor` procs for those types +3. The `toCbor` function provides a convenient way to convert values to CBOR binary data + +For deserialization, the library parses CBOR data into a `CborNode` representation, which can then be converted to Nim values using the `fromCbor` function. This approach allows for flexible handling of CBOR data while maintaining type safety. + +### Current Limitations + +While the JSON implementation supports serde modes via pragmas, the current CBOR implementation does not support the `serialize` and `deserialize` pragmas. The library will raise an assertion error if you try to use these pragmas with CBOR serialization. + +```nim +import pkg/serde/cbor + +type Person {.serialize(mode = OptOut).} = object # This will raise an assertion error + name: string + age: int + isActive: bool +``` + diff --git a/serde/cbor/deserializer.nim b/serde/cbor/deserializer.nim index 64c67a7..55470c1 100644 --- a/serde/cbor/deserializer.nim +++ b/serde/cbor/deserializer.nim @@ -609,7 +609,7 @@ proc fromCbor*[T: object](_: type T, n: CborNode): ?!T = expectCborKind(T, {cborMap}, n) var res = T.default - # Added because serde {serialize, deserialize} pragmas and options are not supported cbor + # Added because serde {serialize, deserialize} pragmas and options are not supported for cbor assertNoPragma(T, deserialize, "deserialize pragma not supported") try: diff --git a/serde/cbor/serializer.nim b/serde/cbor/serializer.nim index bdfefb6..0b24e46 100644 --- a/serde/cbor/serializer.nim +++ b/serde/cbor/serializer.nim @@ -60,20 +60,20 @@ proc writeInitial[T: SomeInteger](str: Stream, m: uint8, n: T): ?!void = str.write(n.uint8) elif uint64(n) <= uint64(uint16.high): str.write(m or 25'u8) - str.write((uint8)n shr 8) - str.write((uint8)n) + str.write((uint8) n shr 8) + str.write((uint8) n) elif uint64(n) <= uint64(uint32.high): str.write(m or 26'u8) for i in countdown(24, 8, 8): {.unroll.} - str.write((uint8)n shr i) - str.write((uint8)n) + str.write((uint8) n shr i) + str.write((uint8) n) else: str.write(m or 27'u8) for i in countdown(56, 8, 8): {.unroll.} - str.write((uint8)n shr i) - str.write((uint8)n) + str.write((uint8) n shr i) + str.write((uint8) n) success() except IOError as e: return failure(e.msg) @@ -200,20 +200,20 @@ proc writeCbor*[T: SomeFloat](str: Stream, v: T): ?!void = return success() of fcZero: str.write initialByte(7, 25) - str.write((char)0x00) + str.write((char) 0x00) of fcNegZero: str.write initialByte(7, 25) - str.write((char)0x80) + str.write((char) 0x80) of fcInf: str.write initialByte(7, 25) - str.write((char)0x7c) + str.write((char) 0x7c) of fcNan: str.write initialByte(7, 25) - str.write((char)0x7e) + str.write((char) 0x7e) of fcNegInf: str.write initialByte(7, 25) - str.write((char)0xfc) - str.write((char)0x00) + str.write((char) 0xfc) + str.write((char) 0x00) success() except IOError as io: return failure(io.msg) @@ -267,7 +267,7 @@ proc writeCbor*(str: Stream, v: CborNode): ?!void = proc writeCbor*[T: object](str: Stream, v: T): ?!void = var n: uint - # Added because serde {serialize, deserialize} pragma and options are not supported cbor + # Added because serde {serialize, deserialize} pragma and options are not supported for cbor assertNoPragma(T, serialize, "serialize pragma not supported") for _, _ in v.fieldPairs: @@ -405,7 +405,7 @@ func initCborBytes*[T: char | byte](buf: openArray[T]): CborNode = ## Create a CBOR byte string from `buf`. result = CborNode(kind: cborBytes, bytes: newSeq[byte](buf.len)) for i in 0 ..< buf.len: - result.bytes[i] = (byte)buf[i] + result.bytes[i] = (byte) buf[i] func initCborBytes*(len: int): CborNode = ## Create a CBOR byte string of ``len`` bytes. diff --git a/serde/json/README.md b/serde/json/README.md new file mode 100644 index 0000000..9aa0af9 --- /dev/null +++ b/serde/json/README.md @@ -0,0 +1,421 @@ +# nim-serde JSON + +Explore JSON serialization and deserialization using nim-serde, an improved alternative to `std/json`. + +## Table of Contents +- [nim-serde JSON](#nim-serde-json) + - [Table of Contents](#table-of-contents) + - [Serialization API](#serialization-api) + - [Basic Serialization with `%` operator](#basic-serialization-with--operator) + - [Object Serialization](#object-serialization) + - [Serialization with `%*`](#serialization-with-) + - [Converting to JSON String with `toJson`](#converting-to-json-string-with-tojson) + - [Serialization Modes](#serialization-modes) + - [Field Customization for Serialization](#field-customization-for-serialization) + - [Deserialization API](#deserialization-api) + - [Basic Deserialization with `fromJson`](#basic-deserialization-with-fromjson) + - [Error Handling](#error-handling) + - [Parsing JSON with `JsonNode.parse`](#parsing-json-with-jsonnodeparse) + - [Deserialization Modes](#deserialization-modes) + - [Field Customization for Deserialization](#field-customization-for-deserialization) + - [Using as a Drop-in Replacement for std/json](#using-as-a-drop-in-replacement-for-stdjson) + - [Custom Type Serialization](#custom-type-serialization) + - [Implementation Details](#implementation-details) + +## Serialization API + +The nim-serde JSON serialization API provides several ways to convert Nim values to JSON. + +### Basic Serialization with `%` operator + +The `%` operator converts Nim values to `JsonNode` objects, which can then be converted to JSON strings: + +```nim +import pkg/serde/json + +# Basic types +assert %42 == newJInt(42) +assert %"hello" == newJString("hello") +assert %true == newJBool(true) + +# Arrays and sequences +let arr = newJArray() +arr.add(newJInt(1)) +arr.add(newJInt(2)) +arr.add(newJInt(3)) +assert $(%[1, 2, 3]) == $arr +``` + +### Object Serialization + +Objects can be serialized using the `%` operator, which automatically handles field serialization based on the object's configuration: + +```nim +import pkg/serde/json + +type Person = object + name {.serialize.}: string + age {.serialize.}: int + address: string # Not serialized by default in OptIn mode + +let person = Person(name: "John", age: 30, address: "123 Main St") +let jsonNode = %person +assert jsonNode.kind == JObject +assert jsonNode.len == 2 +assert jsonNode["name"].getStr == "John" +assert jsonNode["age"].getInt == 30 +assert "address" notin jsonNode +``` + +### Serialization with `%*` + +The `%*` macro provides a more convenient way to create JSON objects: + +```nim +import pkg/serde/json + +let jsonObj = %*{ + "name": "John", + "age": 30, + "hobbies": ["reading", "coding"], + "address": { + "street": "123 Main St", + "city": "Anytown" + } +} + +assert jsonObj.kind == JObject +assert jsonObj["name"].getStr == "John" +assert jsonObj["age"].getInt == 30 +assert jsonObj["hobbies"].kind == JArray +assert jsonObj["hobbies"][0].getStr == "reading" +assert jsonObj["address"]["street"].getStr == "123 Main St" +``` + +### Converting to JSON String with `toJson` + +The `toJson` function converts any serializable value directly to a JSON string: + +```nim +import pkg/serde/json + +type Person = object + name {.serialize.}: string + age {.serialize.}: int + +let person = Person(name: "John", age: 30) +assert person.toJson == """{"name":"John","age":30}""" +``` + +### Serialization Modes + +nim-serde offers three modes to control which fields are serialized: + +```nim +import pkg/serde/json + +# OptIn mode (default): Only fields with {.serialize.} are included +type Person1 = object + name {.serialize.}: string + age {.serialize.}: int + address: string # Not serialized + +assert Person1(name: "John", age: 30, address: "123 Main St").toJson == """{"name":"John","age":30}""" + +# OptOut mode: All fields are included except those marked to ignore +type Person2 {.serialize.} = object + name: string + age: int + ssn {.serialize(ignore=true).}: string # Not serialized + +assert Person2(name: "John", age: 30, ssn: "123-45-6789").toJson == """{"name":"John","age":30}""" + +# Strict mode: All fields are included, and an error is raised if any fields are missing +type Person3 {.serialize(mode=Strict).} = object + name: string + age: int + +assert Person3(name: "John", age: 30).toJson == """{"name":"John","age":30}""" +``` + +### Field Customization for Serialization + +Fields can be customized with various options: + +```nim +import pkg/serde/json + +# Field customization for serialization +type Person {.serialize(mode = OptOut).} = object + firstName {.serialize(key = "first_name").}: string + lastName {.serialize(key = "last_name").}: string + age: int # Will be included because we're using OptOut mode + ssn {.serialize(ignore = true).}: string # Sensitive data not serialized + +let person = Person( + firstName: "John", + lastName: "Doe", + age: 30, + ssn: "123-45-6789" +) + +let jsonNode = %person +assert jsonNode.kind == JObject +assert jsonNode["first_name"].getStr == "John" +assert jsonNode["last_name"].getStr == "Doe" +assert jsonNode["age"].getInt == 30 +assert "ssn" notin jsonNode + +# Convert to JSON string +let jsonStr = toJson(person) +assert jsonStr == """{"first_name":"John","last_name":"Doe","age":30}""" +``` + +## Deserialization API + +nim-serde provides a type-safe way to convert JSON data back into Nim types. + +### Basic Deserialization with `fromJson` + +The `fromJson` function converts JSON strings or `JsonNode` objects to Nim types: + +```nim +import pkg/serde/json +import pkg/questionable/results + +type Person = object + name: string + age: int + +let jsonStr = """{"name":"John","age":30}""" +let result = Person.fromJson(jsonStr) + +# Using the ! operator from questionable to extract the value +assert !result == Person(name: "John", age: 30) +``` + +### Error Handling + +Deserialization returns a `Result` type from the `questionable` library, allowing for safe error handling: + +```nim +import pkg/serde/json +import pkg/questionable/results + +type Person = object + name: string + age: int + +let invalidJson = """{"name":"John","age":"thirty"}""" +let errorResult = Person.fromJson(invalidJson) +assert errorResult.isFailure +assert errorResult.error of UnexpectedKindError +``` + +### Parsing JSON with `JsonNode.parse` + +For parsing raw JSON without immediate deserialization: + +```nim +import pkg/serde/json +import pkg/questionable/results + +let jsonStr = """{"name":"John","age":30,"hobbies":["reading","coding"]}""" +let parseResult = JsonNode.parse(jsonStr) +assert parseResult.isSuccess +let jsonNode = !parseResult +assert jsonNode["name"].getStr == "John" +assert jsonNode["age"].getInt == 30 +assert jsonNode["hobbies"].kind == JArray +assert jsonNode["hobbies"][0].getStr == "reading" +assert jsonNode["hobbies"][1].getStr == "coding" +``` + +### Deserialization Modes + +nim-serde offers three modes to control how JSON is deserialized: + +```nim +import pkg/serde/json +import pkg/questionable/results + +# OptOut mode (default for deserialization) +type PersonOptOut = object + name: string + age: int + +let jsonOptOut = """{"name":"John","age":30,"address":"123 Main St"}""" +let resultOptOut = PersonOptOut.fromJson(jsonOptOut) +assert resultOptOut.isSuccess +assert !resultOptOut == PersonOptOut(name: "John", age: 30) + +# OptIn mode +type PersonOptIn {.deserialize(mode = OptIn).} = object + name {.deserialize.}: string + age {.deserialize.}: int + address: string # Not deserialized by default in OptIn mode + +let jsonOptIn = """{"name":"John","age":30,"address":"123 Main St"}""" +let resultOptIn = PersonOptIn.fromJson(jsonOptIn) +assert resultOptIn.isSuccess +assert (!resultOptIn).name == "John" +assert (!resultOptIn).age == 30 +assert (!resultOptIn).address == "" # address is not deserialized + +# Strict mode +type PersonStrict {.deserialize(mode = Strict).} = object + name: string + age: int + +let jsonStrict = """{"name":"John","age":30}""" +let resultStrict = PersonStrict.fromJson(jsonStrict) +assert resultStrict.isSuccess +assert !resultStrict == PersonStrict(name: "John", age: 30) + +# Strict mode with extra field (should fail) +let jsonStrictExtra = """{"name":"John","age":30,"address":"123 Main St"}""" +let resultStrictExtra = PersonStrict.fromJson(jsonStrictExtra) +assert resultStrictExtra.isFailure +``` + +### Field Customization for Deserialization + +Fields can be customized with various options for deserialization: + +```nim +import pkg/serde/json +import pkg/questionable/results + +type User = object + firstName {.deserialize(key = "first_name").}: string + lastName {.deserialize(key = "last_name").}: string + age: int + internalId {.deserialize(ignore = true).}: int + +let userJsonStr = """{"first_name":"Jane","last_name":"Smith","age":25,"role":"admin"}""" +let result = User.fromJson(userJsonStr) +assert result.isSuccess +assert (!result).firstName == "Jane" +assert (!result).lastName == "Smith" +assert (!result).age == 25 +assert (!result).internalId == 0 # Default value, not deserialized +``` + +## Using as a Drop-in Replacement for std/json + +nim-serde can be used as a drop-in replacement for the standard library's `json` module with improved exception handling. + +```nim +# Instead of: +# import std/json +import pkg/serde/json +import pkg/questionable/results + +# Using nim-serde's JSON API which is compatible with std/json +let jsonNode = %* { + "name": "John", + "age": 30, + "isActive": true, + "hobbies": ["reading", "swimming"] +} + +# Accessing JSON fields using the same API as std/json +assert jsonNode.kind == JObject +assert jsonNode["name"].getStr == "John" +assert jsonNode["age"].getInt == 30 +assert jsonNode["isActive"].getBool == true +assert jsonNode["hobbies"].kind == JArray +assert jsonNode["hobbies"][0].getStr == "reading" +assert jsonNode["hobbies"][1].getStr == "swimming" + +# Converting JSON to string +let jsonStr = $jsonNode + +# Parsing JSON from string with better error handling +let parsedResult = JsonNode.parse(jsonStr) +assert parsedResult.isSuccess +let parsedNode = !parsedResult +assert parsedNode.kind == JObject +assert parsedNode["name"].getStr == "John" + +# Pretty printing +let prettyJson = pretty(jsonNode) +``` + +## Custom Type Serialization + +You can extend nim-serde to support custom types by defining your own `%` operator overloads and `fromJson` procs: + +```nim +import pkg/serde/json +import pkg/serde/utils/errors +import pkg/questionable/results +import std/strutils + +# Define a custom type +type + UserId = distinct int + +# Custom serialization for UserId +proc `%`*(id: UserId): JsonNode = + %("user-" & $int(id)) + +# Custom deserialization for UserId +proc fromJson*(_: type UserId, json: JsonNode): ?!UserId = + if json.kind != JString: + return failure(newSerdeError("Expected string for UserId, got " & $json.kind)) + + let str = json.getStr() + if str.startsWith("user-"): + let idStr = str[5..^1] + try: + let id = parseInt(idStr) + success(UserId(id)) + except ValueError: + failure(newSerdeError("Invalid UserId format: " & str)) + else: + failure(newSerdeError("UserId must start with 'user-' prefix")) + +# Test serialization +let userId = UserId(42) +let jsonNode = %userId +assert jsonNode.kind == JString +assert jsonNode.getStr() == "user-42" + +# Test deserialization +let jsonStr = "\"user-42\"" +let parsedJson = !JsonNode.parse(jsonStr) +let result = UserId.fromJson(parsedJson) +assert result.isSuccess +assert int(!result) == 42 + +# Test in object context +type User {.serialize(mode = OptOut).} = object + id: UserId + name: string + +let user = User(id: UserId(123), name: "John") +let userJson = %user +assert userJson.kind == JObject +assert userJson["id"].getStr() == "user-123" +assert userJson["name"].getStr() == "John" + +# Test deserialization of object with custom type +let userJsonStr = """{"id":"user-123","name":"John"}""" +let userResult = User.fromJson(userJsonStr) +assert userResult.isSuccess +assert int((!userResult).id) == 123 +assert (!userResult).name == "John" +``` + +## Implementation Details + +The JSON serialization in nim-serde is based on the `%` operator pattern: + +1. The `%` operator converts values to `JsonNode` objects +2. Various overloads handle different types (primitives, objects, collections) +3. The `toJson` function converts the `JsonNode` to a string + +This approach makes it easy to extend with custom types by defining your own `%` operator overloads. + +For deserialization, the library uses compile-time reflection to map JSON fields to object fields, respecting the configuration provided by pragmas.