mirror of
https://github.com/logos-storage/nim-serde.git
synced 2026-01-02 13:43:06 +00:00
update readme and add docs for cbor and json wireformats
This commit is contained in:
munna0908
parent
0096796d66
commit
dd00098466
287
README.md
287
README.md
@ -1,102 +1,13 @@
|
||||
# nim-serde
|
||||
|
||||
Easy-to-use json serialization capabilities, and a drop-in replacement for `std/json`.
|
||||
A serialization and deserialization library for Nim supporting multiple wire formats.
|
||||
|
||||
## Quick examples
|
||||
## Supported Wire Formats
|
||||
|
||||
Opt-in serialization by default:
|
||||
nim-serde currently supports the following wire formats:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
type MyType = object
|
||||
field1 {.serialize.}: bool
|
||||
field2: bool
|
||||
|
||||
assert MyType(field1: true, field2: true).toJson == """{"field1":true}"""
|
||||
```
|
||||
|
||||
Opt-out deserialization by default:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
# All fields deserialized, as none are ignored
|
||||
type MyType1 = object
|
||||
field1: bool
|
||||
field2: bool
|
||||
|
||||
let jsn1 = """{
|
||||
"field1": true,
|
||||
"field2": true
|
||||
}"""
|
||||
assert !MyType1.fromJson(jsn1) == MyType1(field1: true, field2: true)
|
||||
|
||||
# Don't deserialize ignored fields in OptOut mode
|
||||
type MyType2 = object
|
||||
field1 {.deserialize(ignore=true).}: bool
|
||||
field2: bool
|
||||
|
||||
let jsn2 = """{
|
||||
"field1": true,
|
||||
"field2": true,
|
||||
"extra": "extra fields don't error in OptOut mode"
|
||||
}"""
|
||||
assert !MyType2.fromJson(jsn2) == MyType2(field1: false, field2: true)
|
||||
|
||||
# Note, the ! operator is part of https://github.com/codex-storage/questionable, which retrieves a value if set
|
||||
```
|
||||
|
||||
Serialize all fields of a type (OptOut mode):
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
type MyType {.serialize.} = object
|
||||
field1: int
|
||||
field2: int
|
||||
|
||||
assert MyType(field1: 1, field2: 2).toJson == """{"field1":1,"field2":2}"""
|
||||
```
|
||||
|
||||
Alias field names in both directions!
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
type MyType {.serialize.} = object
|
||||
field1 {.serialize("othername"),deserialize("takesprecedence").}: int
|
||||
field2: int
|
||||
|
||||
assert MyType(field1: 1, field2: 2).toJson == """{"othername":1,"field2":2}"""
|
||||
let jsn = """{
|
||||
"othername": 1,
|
||||
"field2": 2,
|
||||
"takesprecedence": 3
|
||||
}"""
|
||||
assert !MyType.fromJson(jsn) == MyType(field1: 3, field2: 2)
|
||||
```
|
||||
|
||||
Supports strict mode, where type fields and json fields must match
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
type MyType {.deserialize(mode=Strict).} = object
|
||||
field1: int
|
||||
field2: int
|
||||
|
||||
let jsn = """{
|
||||
"field1": 1,
|
||||
"field2": 2,
|
||||
"extra": 3
|
||||
}"""
|
||||
|
||||
let res = MyType.fromJson(jsn)
|
||||
assert res.isFailure
|
||||
assert res.error of SerdeError
|
||||
assert res.error.msg == "json field(s) missing in object: {\"extra\"}"
|
||||
```
|
||||
- **JSON**: A text-based data interchange format. [See JSON](serde/json/README.md) for details.
|
||||
- **CBOR**: A binary data format following RFC 8949. [See CBOR](serde/cbor/README.md) for details.
|
||||
|
||||
## Serde modes
|
||||
|
||||
@ -215,198 +126,20 @@ Specifying `ignore`, will prevent de/serialization on the field.
|
||||
| `ignore` | <li>**OptOut:** field not serialized</li><li>**OptIn:** field not serialized</li><li>**Strict:** field serialized</li> | <li>**OptOut:** field not deserialized</li><li>**OptIn:** field not deserialized</li><li>**Strict:** field deserialized</li> |
|
||||
|
||||
|
||||
## Deserialization
|
||||
|
||||
`serde` deserializes using `fromJson`, and in all instances returns `Result[T,
|
||||
CatchableError]`, where `T` is the type being deserialized. For example:
|
||||
|
||||
```nim
|
||||
type MyType = object
|
||||
field1: bool
|
||||
field2: bool
|
||||
## Known Issues
|
||||
|
||||
let jsn1 = """{
|
||||
"field1": true,
|
||||
"field2": true
|
||||
}"""
|
||||
There is a known issue when using mixins with generic overloaded procs like `fromJson`. At the time of mixin call, only the `fromJson` overloads in scope of the called mixin are available to be dispatched at runtime. There could be other `fromJson` overloads declared in other modules, but are not in scope at the time the mixin was called.
|
||||
|
||||
assert !MyType.fromJson(jsn1) == MyType(field1: true, field2: true)
|
||||
```
|
||||
Therefore, anytime `fromJson` is called targeting a declared overload, it may or may not be dispatchable. This can be worked around by forcing the `fromJson` overload into scope at compile time. For example, in your application where the `fromJson` overload is defined, at the bottom of the module add:
|
||||
|
||||
If there was an error during deserialization, the result of `fromJson` will contain it:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
type MyType {.deserialize(mode=Strict).} = object
|
||||
field1: int
|
||||
field2: int
|
||||
|
||||
let jsn = """{
|
||||
"field1": 1,
|
||||
"field2": 2,
|
||||
"extra": 3
|
||||
}"""
|
||||
|
||||
let res = MyType.fromJson(jsn)
|
||||
assert res.isFailure
|
||||
assert res.error of SerdeError
|
||||
assert res.error.msg == "json field(s) missing in object: {\"extra\"}"
|
||||
```
|
||||
|
||||
## Custom types
|
||||
|
||||
If `serde` can't de/serialize a custom type, de/serialization can be supported by
|
||||
overloading `%` and `fromJson`. For example:
|
||||
|
||||
```nim
|
||||
type
|
||||
Address* = distinct array[20, byte]
|
||||
SerializationError* = object of CatchableError
|
||||
|
||||
func `%`*(address: Address): JsonNode =
|
||||
%($address)
|
||||
|
||||
func fromJson(_: type Address, json: JsonNode): ?!Address =
|
||||
expectJsonKind(Address, JString, json)
|
||||
without address =? Address.init(json.getStr), error:
|
||||
return failure newException(SerializationError,
|
||||
"Failed to convert '" & $json & "' to Address: " & error.msg)
|
||||
success address
|
||||
```
|
||||
|
||||
## Serializing to string (`toJson`)
|
||||
|
||||
`toJson` is a shortcut for serializing an object into its serialized string
|
||||
representation:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
type MyType {.serialize.} = object
|
||||
field1: string
|
||||
field2: bool
|
||||
|
||||
let mt = MyType(field1: "hw", field2: true)
|
||||
assert mt.toJson == """{"field1":"hw","field2":true}"""
|
||||
```
|
||||
|
||||
This comes in handy, for example, when sending API responses:
|
||||
|
||||
```nim
|
||||
let availability = getAvailability(...)
|
||||
return RestApiResponse.response(availability.toJson,
|
||||
contentType="application/json")
|
||||
```
|
||||
|
||||
## `std/json` drop-in replacment
|
||||
|
||||
`nim-serde` can be used as a drop-in replacement for the [standard library's `json`
|
||||
module](https://nim-lang.org/docs/json.html), with a few notable improvements.
|
||||
|
||||
Instead of importing `std/json` into your application, `pkg/serde/json` can be imported
|
||||
instead:
|
||||
|
||||
```diff
|
||||
- import std/json
|
||||
+ import pkg/serde/json
|
||||
```
|
||||
|
||||
As with `std/json`, `%` can be used to serialize a type into a `JsonNode`:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
assert %"hello" == newJString("hello")
|
||||
```
|
||||
|
||||
And `%*` can be used to serialize objects:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
let expected = newJObject()
|
||||
expected["hello"] = newJString("world")
|
||||
assert %*{"hello": "world"} == expected
|
||||
```
|
||||
|
||||
As well, serialization of types can be overridden, and serialization of custom types can
|
||||
be introduced. Here, we are overriding the serialization of `int`:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
func `%`(i: int): JsonNode =
|
||||
newJInt(i + 1)
|
||||
|
||||
assert 1.toJson == "2"
|
||||
```
|
||||
|
||||
## `parseJson` and exception tracking
|
||||
|
||||
Unfortunately, `std/json`'s `parseJson` can raise an `Exception`, so proper exception
|
||||
tracking breaks, eg
|
||||
|
||||
```nim
|
||||
|
||||
## Fails to compile:
|
||||
## Error: parseJson(me, false, false) can raise an unlisted exception: Exception
|
||||
|
||||
import std/json
|
||||
|
||||
{.push raises:[].}
|
||||
|
||||
type
|
||||
MyAppError = object of CatchableError
|
||||
|
||||
proc parseMe(me: string): JsonNode =
|
||||
try:
|
||||
return me.parseJson
|
||||
except CatchableError as error:
|
||||
raise newException(MyAppError, error.msg)
|
||||
|
||||
assert """{"hello":"world"}""".parseMe == %* { "hello": "world" }
|
||||
```
|
||||
|
||||
This is due to `std/json`'s `parseJson` incorrectly raising `Exception`. This can be
|
||||
worked around by instead importing `serde` and calling its `JsonNode.parse` routine.
|
||||
Note that `serde`'s `JsonNode.parse` returns a `Result[JsonNode, CatchableError]`
|
||||
instead of just a plain `JsonNode` object as in `std/json`'s `parseJson`:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
{.push raises:[].}
|
||||
|
||||
type
|
||||
MyAppError = object of CatchableError
|
||||
|
||||
proc parseMe(me: string): JsonNode {.raises: [MyAppError].} =
|
||||
without parsed =? JsonNode.parse(me), error:
|
||||
raise newException(MyAppError, error.msg)
|
||||
parsed
|
||||
|
||||
assert """{"hello":"world"}""".parseMe == %* { "hello": "world" }
|
||||
```
|
||||
|
||||
## Known issues
|
||||
|
||||
There is a known issue when using mixins with generic overloaded procs like
|
||||
`fromJson`. At the time of mixin call, only the `fromJson` overloads in scope of
|
||||
the called mixin are available to be dispatched at runtime. There could be other
|
||||
`fromJson` overloads declared in other modules, but are not in scope at the time
|
||||
the mixin was called. Therefore, anytime `fromJson` is called targeting a
|
||||
declared overload, it may or may not be dispatchable. This can be worked around
|
||||
by forcing the `fromJson` overload into scope at compile time. For example, in
|
||||
your application where the `fromJson` overload is defined, at the bottom of the
|
||||
module add:
|
||||
```nim
|
||||
static: MyType.fromJson("")
|
||||
```
|
||||
|
||||
This will ensure that the `MyType.fromJson` overload is dispatchable.
|
||||
|
||||
The basic types that serde supports should already have their overloads forced
|
||||
in scope in [the `deserializer`
|
||||
module](./serde/json/deserializer.nim#L340-L356).
|
||||
The basic types that serde supports should already have their overloads forced in scope in [the `deserializer` module](./serde/json/deserializer.nim#L340-L356).
|
||||
|
||||
For an illustration of the problem, please see this [narrow example](https://github.com/gmega/serialization-bug/tree/main/narrow) by [@gmega](https://github.com/gmega).
|
||||
|
||||
376
serde/cbor/README.md
Normal file
376
serde/cbor/README.md
Normal file
@ -0,0 +1,376 @@
|
||||
# nim-serde CBOR
|
||||
|
||||
This README details the usage of CBOR serialization and deserialization features offered by nim-serde, in compliance with [RFC 8949](https://datatracker.ietf.org/doc/html/rfc8949).
|
||||
|
||||
## Table of Contents
|
||||
- [nim-serde CBOR](#nim-serde-cbor)
|
||||
- [Table of Contents](#table-of-contents)
|
||||
- [Serialization API](#serialization-api)
|
||||
- [Basic Serialization with Stream API](#basic-serialization-with-stream-api)
|
||||
- [Object Serialization](#object-serialization)
|
||||
- [Custom Type Serialization](#custom-type-serialization)
|
||||
- [Converting to CBOR with `toCbor`](#converting-to-cbor-with-tocbor)
|
||||
- [Working with CborNode](#working-with-cbornode)
|
||||
- [Convenience Functions for CborNode](#convenience-functions-for-cbornode)
|
||||
- [Deserialization API](#deserialization-api)
|
||||
- [Basic Deserialization with `fromCbor`](#basic-deserialization-with-fromcbor)
|
||||
- [Error Handling](#error-handling)
|
||||
- [Parsing CBOR with `parseCbor`](#parsing-cbor-with-parsecbor)
|
||||
- [Custom Type Deserialization](#custom-type-deserialization)
|
||||
- [Implementation Details](#implementation-details)
|
||||
- [Current Limitations](#current-limitations)
|
||||
|
||||
## Serialization API
|
||||
|
||||
The nim-serde CBOR serialization API provides several ways to convert Nim values to CBOR.
|
||||
|
||||
### Basic Serialization with Stream API
|
||||
|
||||
The `writeCbor` function writes Nim values to a stream in CBOR format:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
import std/streams
|
||||
|
||||
# Create a stream to write to
|
||||
let stream = newStringStream()
|
||||
|
||||
# Basic types
|
||||
discard stream.writeCbor(42) # Unsigned integer
|
||||
discard stream.writeCbor(-10) # Negative integer
|
||||
discard stream.writeCbor(3.14) # Float
|
||||
discard stream.writeCbor("hello") # String
|
||||
discard stream.writeCbor(true) # Boolean
|
||||
|
||||
# Arrays and sequences
|
||||
discard stream.writeCbor(@[1, 2, 3]) # Sequence
|
||||
|
||||
# Get the serialized CBOR data
|
||||
let cborData = stream.data
|
||||
```
|
||||
|
||||
### Object Serialization
|
||||
|
||||
Objects can be serialized to CBOR format using the stream API:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
import std/streams
|
||||
|
||||
type Person = object
|
||||
name: string
|
||||
age: int
|
||||
isActive: bool
|
||||
|
||||
let person = Person(
|
||||
name: "John",
|
||||
age: 30,
|
||||
isActive: true
|
||||
)
|
||||
|
||||
# Serialize the object to CBOR
|
||||
let stream = newStringStream()
|
||||
discard stream.writeCbor(person)
|
||||
|
||||
# Get the serialized CBOR data
|
||||
let cborData = stream.data
|
||||
```
|
||||
|
||||
### Custom Type Serialization
|
||||
|
||||
You can extend nim-serde to support custom types by defining your own `writeCbor` procs:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
import std/streams
|
||||
import std/strutils
|
||||
|
||||
# Define a custom type
|
||||
type
|
||||
UserId = distinct int
|
||||
|
||||
# Custom serialization for UserId
|
||||
proc writeCbor*(str: Stream, id: UserId): ?!void =
|
||||
# Write as a CBOR text string with a prefix
|
||||
str.writeCbor("user-" & $int(id))
|
||||
|
||||
# Test serialization
|
||||
let userId = UserId(42)
|
||||
let stream = newStringStream()
|
||||
discard stream.writeCbor(userId)
|
||||
let cborData = stream.data
|
||||
|
||||
# Test in object context
|
||||
type User = object
|
||||
id: UserId
|
||||
name: string
|
||||
|
||||
let user = User(id: UserId(123), name: "John")
|
||||
let userStream = newStringStream()
|
||||
discard userStream.writeCbor(user)
|
||||
let userCborData = userStream.data
|
||||
```
|
||||
|
||||
### Converting to CBOR with `toCbor`
|
||||
|
||||
The `toCbor` function can be used to directly convert a Nim value to CBOR binary data:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
|
||||
type Person = object
|
||||
name: string
|
||||
age: int
|
||||
isActive: bool
|
||||
|
||||
let person = Person(
|
||||
name: "John",
|
||||
age: 30,
|
||||
isActive: true
|
||||
)
|
||||
|
||||
# Convert to CBOR binary data
|
||||
let result = toCbor(person)
|
||||
assert result.isSuccess
|
||||
let cborData = !result
|
||||
```
|
||||
|
||||
### Working with CborNode
|
||||
|
||||
The `CborNode` type represents CBOR data in memory and can be manipulated directly:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
import std/tables
|
||||
|
||||
# Create CBOR nodes
|
||||
let textNode = CborNode(kind: cborText, text: "hello")
|
||||
let intNode = CborNode(kind: cborUnsigned, uint: 42'u64)
|
||||
let floatNode = CborNode(kind: cborFloat, float: 3.14)
|
||||
|
||||
# Create an array
|
||||
var arrayNode = CborNode(kind: cborArray)
|
||||
arrayNode.seq = @[textNode, intNode, floatNode]
|
||||
|
||||
# Create a map with text keys and boolean values
|
||||
var mapNode = CborNode(kind: cborMap)
|
||||
mapNode.map = initOrderedTable[CborNode, CborNode]()
|
||||
# Boolean values are represented as simple values (21 for true, 20 for false)
|
||||
mapNode.map[CborNode(kind: cborText, text: "a")] = CborNode(kind: cborSimple, simple: 21) # true
|
||||
mapNode.map[CborNode(kind: cborText, text: "b")] = CborNode(kind: cborSimple, simple: 20) # false
|
||||
|
||||
# Convert to CBOR binary data
|
||||
let result = toCbor(mapNode)
|
||||
assert result.isSuccess
|
||||
let cborData = !result
|
||||
```
|
||||
|
||||
### Convenience Functions for CborNode
|
||||
|
||||
The library provides convenience functions for creating CBOR nodes:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
|
||||
# Initialize CBOR nodes
|
||||
let bytesNode = initCborBytes(@[byte 1, byte 2, byte 3])
|
||||
let textNode = initCborText("hello")
|
||||
let arrayNode = initCborArray()
|
||||
let mapNode = initCborMap()
|
||||
|
||||
# Convert values to CborNode
|
||||
let intNodeResult = toCborNode(42)
|
||||
assert intNodeResult.isSuccess
|
||||
let intNode = !intNodeResult
|
||||
|
||||
let strNodeResult = toCborNode("hello")
|
||||
assert strNodeResult.isSuccess
|
||||
let strNode = !strNodeResult
|
||||
|
||||
let boolNodeResult = toCborNode(true)
|
||||
assert boolNodeResult.isSuccess
|
||||
let boolNode = !boolNodeResult
|
||||
```
|
||||
|
||||
## Deserialization API
|
||||
|
||||
The nim-serde CBOR deserialization API provides ways to convert CBOR data back to Nim values.
|
||||
|
||||
### Basic Deserialization with `fromCbor`
|
||||
|
||||
The `fromCbor` function converts CBOR data to Nim values:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
import std/streams
|
||||
|
||||
# Create some CBOR data
|
||||
let stream = newStringStream()
|
||||
discard stream.writeCbor(42)
|
||||
let cborData = stream.data
|
||||
|
||||
# Parse the CBOR data into a CborNode
|
||||
try:
|
||||
let node = parseCbor(cborData)
|
||||
|
||||
# Deserialize the CborNode to a Nim value
|
||||
let intResult = int.fromCbor(node)
|
||||
assert intResult.isSuccess
|
||||
let value = !intResult
|
||||
assert value == 42
|
||||
|
||||
# You can also deserialize to other types
|
||||
# For example, if cborData contained a string:
|
||||
# let strResult = string.fromCbor(node)
|
||||
# assert strResult.isSuccess
|
||||
# let strValue = !strResult
|
||||
|
||||
# Deserialize to an object
|
||||
type Person = object
|
||||
name: string
|
||||
age: int
|
||||
isActive: bool
|
||||
|
||||
let personResult = Person.fromCbor(node)
|
||||
assert personResult.isSuccess
|
||||
let person = !personResult
|
||||
|
||||
# Verify the deserialized data
|
||||
assert person.name == "John"
|
||||
assert person.age == 30
|
||||
assert person.isActive == true
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
Deserialization returns a `Result` type from the `questionable` library, allowing for safe error handling:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
|
||||
# Invalid CBOR data for an integer
|
||||
let invalidNode = CborNode(kind: cborText, text: "not an int")
|
||||
let result = int.fromCbor(invalidNode)
|
||||
|
||||
# Check for failure
|
||||
assert result.isFailure
|
||||
echo result.error.msg
|
||||
# Output: "deserialization to int failed: expected {cborUnsigned, cborNegative} but got cborText"
|
||||
```
|
||||
|
||||
### Parsing CBOR with `parseCbor`
|
||||
|
||||
The `parseCbor` function parses CBOR binary data into a `CborNode`:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
|
||||
# Parse CBOR data
|
||||
let node = parseCbor(cborData)
|
||||
|
||||
# Check node type and access data
|
||||
case node.kind
|
||||
of cborUnsigned:
|
||||
echo "Unsigned integer: ", node.uint
|
||||
of cborNegative:
|
||||
echo "Negative integer: ", node.int
|
||||
of cborText:
|
||||
echo "Text: ", node.text
|
||||
of cborArray:
|
||||
echo "Array with ", node.seq.len, " items"
|
||||
of cborMap:
|
||||
echo "Map with ", node.map.len, " pairs"
|
||||
else:
|
||||
echo "Other CBOR type: ", node.kind
|
||||
```
|
||||
|
||||
### Custom Type Deserialization
|
||||
|
||||
You can extend nim-serde to support custom type deserialization by defining your own `fromCbor` procs:
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
import pkg/questionable/results
|
||||
import std/strutils
|
||||
|
||||
# Define a custom type
|
||||
type
|
||||
UserId = distinct int
|
||||
|
||||
# Custom deserialization for UserId
|
||||
proc fromCbor*(_: type UserId, n: CborNode): ?!UserId =
|
||||
if n.kind != cborText:
|
||||
return failure(newSerdeError("Expected string for UserId, got " & $n.kind))
|
||||
|
||||
let str = n.text
|
||||
if str.startsWith("user-"):
|
||||
let idStr = str[5..^1]
|
||||
try:
|
||||
let id = parseInt(idStr)
|
||||
success(UserId(id))
|
||||
except ValueError:
|
||||
failure(newSerdeError("Invalid UserId format: " & str))
|
||||
else:
|
||||
failure(newSerdeError("UserId must start with 'user-' prefix"))
|
||||
|
||||
# Test deserialization
|
||||
let node = parseCbor(cborData) # Assuming cborData contains a serialized UserId
|
||||
let result = UserId.fromCbor(node)
|
||||
assert result.isSuccess
|
||||
assert int(!result) == 42
|
||||
|
||||
# Test deserialization in object context
|
||||
type User = object
|
||||
id: UserId
|
||||
name: string
|
||||
|
||||
let userNode = parseCbor(userCborData) # Assuming userCborData contains a serialized User
|
||||
let userResult = User.fromCbor(userNode)
|
||||
assert userResult.isSuccess
|
||||
assert int((!userResult).id) == 123
|
||||
assert (!userResult).name == "John"
|
||||
```
|
||||
|
||||
|
||||
## Implementation Details
|
||||
|
||||
The CBOR serialization in nim-serde follows a stream-based approach:
|
||||
|
||||
```
|
||||
# Serialization flow
|
||||
Nim value → writeCbor → CBOR binary data
|
||||
|
||||
# Deserialization flow
|
||||
CBOR binary data → parseCbor (CborNode) → fromCbor → Nim value
|
||||
```
|
||||
|
||||
Unlike the JSON implementation which uses the `%` operator pattern, the CBOR implementation uses a hook-based approach:
|
||||
|
||||
1. The `writeCbor` function writes Nim values directly to a stream in CBOR format
|
||||
2. Custom types can be supported by defining `writeCbor` procs for those types
|
||||
3. The `toCbor` function provides a convenient way to convert values to CBOR binary data
|
||||
|
||||
For deserialization, the library parses CBOR data into a `CborNode` representation, which can then be converted to Nim values using the `fromCbor` function. This approach allows for flexible handling of CBOR data while maintaining type safety.
|
||||
|
||||
### Current Limitations
|
||||
|
||||
While the JSON implementation supports serde modes via pragmas, the current CBOR implementation does not support the `serialize` and `deserialize` pragmas. The library will raise an assertion error if you try to use these pragmas with CBOR serialization.
|
||||
|
||||
```nim
|
||||
import pkg/serde/cbor
|
||||
|
||||
type Person {.serialize(mode = OptOut).} = object # This will raise an assertion error
|
||||
name: string
|
||||
age: int
|
||||
isActive: bool
|
||||
```
|
||||
|
||||
@ -609,7 +609,7 @@ proc fromCbor*[T: object](_: type T, n: CborNode): ?!T =
|
||||
expectCborKind(T, {cborMap}, n)
|
||||
var res = T.default
|
||||
|
||||
# Added because serde {serialize, deserialize} pragmas and options are not supported cbor
|
||||
# Added because serde {serialize, deserialize} pragmas and options are not supported for cbor
|
||||
assertNoPragma(T, deserialize, "deserialize pragma not supported")
|
||||
|
||||
try:
|
||||
|
||||
@ -60,20 +60,20 @@ proc writeInitial[T: SomeInteger](str: Stream, m: uint8, n: T): ?!void =
|
||||
str.write(n.uint8)
|
||||
elif uint64(n) <= uint64(uint16.high):
|
||||
str.write(m or 25'u8)
|
||||
str.write((uint8)n shr 8)
|
||||
str.write((uint8)n)
|
||||
str.write((uint8) n shr 8)
|
||||
str.write((uint8) n)
|
||||
elif uint64(n) <= uint64(uint32.high):
|
||||
str.write(m or 26'u8)
|
||||
for i in countdown(24, 8, 8):
|
||||
{.unroll.}
|
||||
str.write((uint8)n shr i)
|
||||
str.write((uint8)n)
|
||||
str.write((uint8) n shr i)
|
||||
str.write((uint8) n)
|
||||
else:
|
||||
str.write(m or 27'u8)
|
||||
for i in countdown(56, 8, 8):
|
||||
{.unroll.}
|
||||
str.write((uint8)n shr i)
|
||||
str.write((uint8)n)
|
||||
str.write((uint8) n shr i)
|
||||
str.write((uint8) n)
|
||||
success()
|
||||
except IOError as e:
|
||||
return failure(e.msg)
|
||||
@ -200,20 +200,20 @@ proc writeCbor*[T: SomeFloat](str: Stream, v: T): ?!void =
|
||||
return success()
|
||||
of fcZero:
|
||||
str.write initialByte(7, 25)
|
||||
str.write((char)0x00)
|
||||
str.write((char) 0x00)
|
||||
of fcNegZero:
|
||||
str.write initialByte(7, 25)
|
||||
str.write((char)0x80)
|
||||
str.write((char) 0x80)
|
||||
of fcInf:
|
||||
str.write initialByte(7, 25)
|
||||
str.write((char)0x7c)
|
||||
str.write((char) 0x7c)
|
||||
of fcNan:
|
||||
str.write initialByte(7, 25)
|
||||
str.write((char)0x7e)
|
||||
str.write((char) 0x7e)
|
||||
of fcNegInf:
|
||||
str.write initialByte(7, 25)
|
||||
str.write((char)0xfc)
|
||||
str.write((char)0x00)
|
||||
str.write((char) 0xfc)
|
||||
str.write((char) 0x00)
|
||||
success()
|
||||
except IOError as io:
|
||||
return failure(io.msg)
|
||||
@ -267,7 +267,7 @@ proc writeCbor*(str: Stream, v: CborNode): ?!void =
|
||||
|
||||
proc writeCbor*[T: object](str: Stream, v: T): ?!void =
|
||||
var n: uint
|
||||
# Added because serde {serialize, deserialize} pragma and options are not supported cbor
|
||||
# Added because serde {serialize, deserialize} pragma and options are not supported for cbor
|
||||
assertNoPragma(T, serialize, "serialize pragma not supported")
|
||||
|
||||
for _, _ in v.fieldPairs:
|
||||
@ -405,7 +405,7 @@ func initCborBytes*[T: char | byte](buf: openArray[T]): CborNode =
|
||||
## Create a CBOR byte string from `buf`.
|
||||
result = CborNode(kind: cborBytes, bytes: newSeq[byte](buf.len))
|
||||
for i in 0 ..< buf.len:
|
||||
result.bytes[i] = (byte)buf[i]
|
||||
result.bytes[i] = (byte) buf[i]
|
||||
|
||||
func initCborBytes*(len: int): CborNode =
|
||||
## Create a CBOR byte string of ``len`` bytes.
|
||||
|
||||
421
serde/json/README.md
Normal file
421
serde/json/README.md
Normal file
@ -0,0 +1,421 @@
|
||||
# nim-serde JSON
|
||||
|
||||
Explore JSON serialization and deserialization using nim-serde, an improved alternative to `std/json`.
|
||||
|
||||
## Table of Contents
|
||||
- [nim-serde JSON](#nim-serde-json)
|
||||
- [Table of Contents](#table-of-contents)
|
||||
- [Serialization API](#serialization-api)
|
||||
- [Basic Serialization with `%` operator](#basic-serialization-with--operator)
|
||||
- [Object Serialization](#object-serialization)
|
||||
- [Serialization with `%*`](#serialization-with-)
|
||||
- [Converting to JSON String with `toJson`](#converting-to-json-string-with-tojson)
|
||||
- [Serialization Modes](#serialization-modes)
|
||||
- [Field Customization for Serialization](#field-customization-for-serialization)
|
||||
- [Deserialization API](#deserialization-api)
|
||||
- [Basic Deserialization with `fromJson`](#basic-deserialization-with-fromjson)
|
||||
- [Error Handling](#error-handling)
|
||||
- [Parsing JSON with `JsonNode.parse`](#parsing-json-with-jsonnodeparse)
|
||||
- [Deserialization Modes](#deserialization-modes)
|
||||
- [Field Customization for Deserialization](#field-customization-for-deserialization)
|
||||
- [Using as a Drop-in Replacement for std/json](#using-as-a-drop-in-replacement-for-stdjson)
|
||||
- [Custom Type Serialization](#custom-type-serialization)
|
||||
- [Implementation Details](#implementation-details)
|
||||
|
||||
## Serialization API
|
||||
|
||||
The nim-serde JSON serialization API provides several ways to convert Nim values to JSON.
|
||||
|
||||
### Basic Serialization with `%` operator
|
||||
|
||||
The `%` operator converts Nim values to `JsonNode` objects, which can then be converted to JSON strings:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
# Basic types
|
||||
assert %42 == newJInt(42)
|
||||
assert %"hello" == newJString("hello")
|
||||
assert %true == newJBool(true)
|
||||
|
||||
# Arrays and sequences
|
||||
let arr = newJArray()
|
||||
arr.add(newJInt(1))
|
||||
arr.add(newJInt(2))
|
||||
arr.add(newJInt(3))
|
||||
assert $(%[1, 2, 3]) == $arr
|
||||
```
|
||||
|
||||
### Object Serialization
|
||||
|
||||
Objects can be serialized using the `%` operator, which automatically handles field serialization based on the object's configuration:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
type Person = object
|
||||
name {.serialize.}: string
|
||||
age {.serialize.}: int
|
||||
address: string # Not serialized by default in OptIn mode
|
||||
|
||||
let person = Person(name: "John", age: 30, address: "123 Main St")
|
||||
let jsonNode = %person
|
||||
assert jsonNode.kind == JObject
|
||||
assert jsonNode.len == 2
|
||||
assert jsonNode["name"].getStr == "John"
|
||||
assert jsonNode["age"].getInt == 30
|
||||
assert "address" notin jsonNode
|
||||
```
|
||||
|
||||
### Serialization with `%*`
|
||||
|
||||
The `%*` macro provides a more convenient way to create JSON objects:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
let jsonObj = %*{
|
||||
"name": "John",
|
||||
"age": 30,
|
||||
"hobbies": ["reading", "coding"],
|
||||
"address": {
|
||||
"street": "123 Main St",
|
||||
"city": "Anytown"
|
||||
}
|
||||
}
|
||||
|
||||
assert jsonObj.kind == JObject
|
||||
assert jsonObj["name"].getStr == "John"
|
||||
assert jsonObj["age"].getInt == 30
|
||||
assert jsonObj["hobbies"].kind == JArray
|
||||
assert jsonObj["hobbies"][0].getStr == "reading"
|
||||
assert jsonObj["address"]["street"].getStr == "123 Main St"
|
||||
```
|
||||
|
||||
### Converting to JSON String with `toJson`
|
||||
|
||||
The `toJson` function converts any serializable value directly to a JSON string:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
type Person = object
|
||||
name {.serialize.}: string
|
||||
age {.serialize.}: int
|
||||
|
||||
let person = Person(name: "John", age: 30)
|
||||
assert person.toJson == """{"name":"John","age":30}"""
|
||||
```
|
||||
|
||||
### Serialization Modes
|
||||
|
||||
nim-serde offers three modes to control which fields are serialized:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
# OptIn mode (default): Only fields with {.serialize.} are included
|
||||
type Person1 = object
|
||||
name {.serialize.}: string
|
||||
age {.serialize.}: int
|
||||
address: string # Not serialized
|
||||
|
||||
assert Person1(name: "John", age: 30, address: "123 Main St").toJson == """{"name":"John","age":30}"""
|
||||
|
||||
# OptOut mode: All fields are included except those marked to ignore
|
||||
type Person2 {.serialize.} = object
|
||||
name: string
|
||||
age: int
|
||||
ssn {.serialize(ignore=true).}: string # Not serialized
|
||||
|
||||
assert Person2(name: "John", age: 30, ssn: "123-45-6789").toJson == """{"name":"John","age":30}"""
|
||||
|
||||
# Strict mode: All fields are included, and an error is raised if any fields are missing
|
||||
type Person3 {.serialize(mode=Strict).} = object
|
||||
name: string
|
||||
age: int
|
||||
|
||||
assert Person3(name: "John", age: 30).toJson == """{"name":"John","age":30}"""
|
||||
```
|
||||
|
||||
### Field Customization for Serialization
|
||||
|
||||
Fields can be customized with various options:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
|
||||
# Field customization for serialization
|
||||
type Person {.serialize(mode = OptOut).} = object
|
||||
firstName {.serialize(key = "first_name").}: string
|
||||
lastName {.serialize(key = "last_name").}: string
|
||||
age: int # Will be included because we're using OptOut mode
|
||||
ssn {.serialize(ignore = true).}: string # Sensitive data not serialized
|
||||
|
||||
let person = Person(
|
||||
firstName: "John",
|
||||
lastName: "Doe",
|
||||
age: 30,
|
||||
ssn: "123-45-6789"
|
||||
)
|
||||
|
||||
let jsonNode = %person
|
||||
assert jsonNode.kind == JObject
|
||||
assert jsonNode["first_name"].getStr == "John"
|
||||
assert jsonNode["last_name"].getStr == "Doe"
|
||||
assert jsonNode["age"].getInt == 30
|
||||
assert "ssn" notin jsonNode
|
||||
|
||||
# Convert to JSON string
|
||||
let jsonStr = toJson(person)
|
||||
assert jsonStr == """{"first_name":"John","last_name":"Doe","age":30}"""
|
||||
```
|
||||
|
||||
## Deserialization API
|
||||
|
||||
nim-serde provides a type-safe way to convert JSON data back into Nim types.
|
||||
|
||||
### Basic Deserialization with `fromJson`
|
||||
|
||||
The `fromJson` function converts JSON strings or `JsonNode` objects to Nim types:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
import pkg/questionable/results
|
||||
|
||||
type Person = object
|
||||
name: string
|
||||
age: int
|
||||
|
||||
let jsonStr = """{"name":"John","age":30}"""
|
||||
let result = Person.fromJson(jsonStr)
|
||||
|
||||
# Using the ! operator from questionable to extract the value
|
||||
assert !result == Person(name: "John", age: 30)
|
||||
```
|
||||
|
||||
### Error Handling
|
||||
|
||||
Deserialization returns a `Result` type from the `questionable` library, allowing for safe error handling:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
import pkg/questionable/results
|
||||
|
||||
type Person = object
|
||||
name: string
|
||||
age: int
|
||||
|
||||
let invalidJson = """{"name":"John","age":"thirty"}"""
|
||||
let errorResult = Person.fromJson(invalidJson)
|
||||
assert errorResult.isFailure
|
||||
assert errorResult.error of UnexpectedKindError
|
||||
```
|
||||
|
||||
### Parsing JSON with `JsonNode.parse`
|
||||
|
||||
For parsing raw JSON without immediate deserialization:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
import pkg/questionable/results
|
||||
|
||||
let jsonStr = """{"name":"John","age":30,"hobbies":["reading","coding"]}"""
|
||||
let parseResult = JsonNode.parse(jsonStr)
|
||||
assert parseResult.isSuccess
|
||||
let jsonNode = !parseResult
|
||||
assert jsonNode["name"].getStr == "John"
|
||||
assert jsonNode["age"].getInt == 30
|
||||
assert jsonNode["hobbies"].kind == JArray
|
||||
assert jsonNode["hobbies"][0].getStr == "reading"
|
||||
assert jsonNode["hobbies"][1].getStr == "coding"
|
||||
```
|
||||
|
||||
### Deserialization Modes
|
||||
|
||||
nim-serde offers three modes to control how JSON is deserialized:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
import pkg/questionable/results
|
||||
|
||||
# OptOut mode (default for deserialization)
|
||||
type PersonOptOut = object
|
||||
name: string
|
||||
age: int
|
||||
|
||||
let jsonOptOut = """{"name":"John","age":30,"address":"123 Main St"}"""
|
||||
let resultOptOut = PersonOptOut.fromJson(jsonOptOut)
|
||||
assert resultOptOut.isSuccess
|
||||
assert !resultOptOut == PersonOptOut(name: "John", age: 30)
|
||||
|
||||
# OptIn mode
|
||||
type PersonOptIn {.deserialize(mode = OptIn).} = object
|
||||
name {.deserialize.}: string
|
||||
age {.deserialize.}: int
|
||||
address: string # Not deserialized by default in OptIn mode
|
||||
|
||||
let jsonOptIn = """{"name":"John","age":30,"address":"123 Main St"}"""
|
||||
let resultOptIn = PersonOptIn.fromJson(jsonOptIn)
|
||||
assert resultOptIn.isSuccess
|
||||
assert (!resultOptIn).name == "John"
|
||||
assert (!resultOptIn).age == 30
|
||||
assert (!resultOptIn).address == "" # address is not deserialized
|
||||
|
||||
# Strict mode
|
||||
type PersonStrict {.deserialize(mode = Strict).} = object
|
||||
name: string
|
||||
age: int
|
||||
|
||||
let jsonStrict = """{"name":"John","age":30}"""
|
||||
let resultStrict = PersonStrict.fromJson(jsonStrict)
|
||||
assert resultStrict.isSuccess
|
||||
assert !resultStrict == PersonStrict(name: "John", age: 30)
|
||||
|
||||
# Strict mode with extra field (should fail)
|
||||
let jsonStrictExtra = """{"name":"John","age":30,"address":"123 Main St"}"""
|
||||
let resultStrictExtra = PersonStrict.fromJson(jsonStrictExtra)
|
||||
assert resultStrictExtra.isFailure
|
||||
```
|
||||
|
||||
### Field Customization for Deserialization
|
||||
|
||||
Fields can be customized with various options for deserialization:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
import pkg/questionable/results
|
||||
|
||||
type User = object
|
||||
firstName {.deserialize(key = "first_name").}: string
|
||||
lastName {.deserialize(key = "last_name").}: string
|
||||
age: int
|
||||
internalId {.deserialize(ignore = true).}: int
|
||||
|
||||
let userJsonStr = """{"first_name":"Jane","last_name":"Smith","age":25,"role":"admin"}"""
|
||||
let result = User.fromJson(userJsonStr)
|
||||
assert result.isSuccess
|
||||
assert (!result).firstName == "Jane"
|
||||
assert (!result).lastName == "Smith"
|
||||
assert (!result).age == 25
|
||||
assert (!result).internalId == 0 # Default value, not deserialized
|
||||
```
|
||||
|
||||
## Using as a Drop-in Replacement for std/json
|
||||
|
||||
nim-serde can be used as a drop-in replacement for the standard library's `json` module with improved exception handling.
|
||||
|
||||
```nim
|
||||
# Instead of:
|
||||
# import std/json
|
||||
import pkg/serde/json
|
||||
import pkg/questionable/results
|
||||
|
||||
# Using nim-serde's JSON API which is compatible with std/json
|
||||
let jsonNode = %* {
|
||||
"name": "John",
|
||||
"age": 30,
|
||||
"isActive": true,
|
||||
"hobbies": ["reading", "swimming"]
|
||||
}
|
||||
|
||||
# Accessing JSON fields using the same API as std/json
|
||||
assert jsonNode.kind == JObject
|
||||
assert jsonNode["name"].getStr == "John"
|
||||
assert jsonNode["age"].getInt == 30
|
||||
assert jsonNode["isActive"].getBool == true
|
||||
assert jsonNode["hobbies"].kind == JArray
|
||||
assert jsonNode["hobbies"][0].getStr == "reading"
|
||||
assert jsonNode["hobbies"][1].getStr == "swimming"
|
||||
|
||||
# Converting JSON to string
|
||||
let jsonStr = $jsonNode
|
||||
|
||||
# Parsing JSON from string with better error handling
|
||||
let parsedResult = JsonNode.parse(jsonStr)
|
||||
assert parsedResult.isSuccess
|
||||
let parsedNode = !parsedResult
|
||||
assert parsedNode.kind == JObject
|
||||
assert parsedNode["name"].getStr == "John"
|
||||
|
||||
# Pretty printing
|
||||
let prettyJson = pretty(jsonNode)
|
||||
```
|
||||
|
||||
## Custom Type Serialization
|
||||
|
||||
You can extend nim-serde to support custom types by defining your own `%` operator overloads and `fromJson` procs:
|
||||
|
||||
```nim
|
||||
import pkg/serde/json
|
||||
import pkg/serde/utils/errors
|
||||
import pkg/questionable/results
|
||||
import std/strutils
|
||||
|
||||
# Define a custom type
|
||||
type
|
||||
UserId = distinct int
|
||||
|
||||
# Custom serialization for UserId
|
||||
proc `%`*(id: UserId): JsonNode =
|
||||
%("user-" & $int(id))
|
||||
|
||||
# Custom deserialization for UserId
|
||||
proc fromJson*(_: type UserId, json: JsonNode): ?!UserId =
|
||||
if json.kind != JString:
|
||||
return failure(newSerdeError("Expected string for UserId, got " & $json.kind))
|
||||
|
||||
let str = json.getStr()
|
||||
if str.startsWith("user-"):
|
||||
let idStr = str[5..^1]
|
||||
try:
|
||||
let id = parseInt(idStr)
|
||||
success(UserId(id))
|
||||
except ValueError:
|
||||
failure(newSerdeError("Invalid UserId format: " & str))
|
||||
else:
|
||||
failure(newSerdeError("UserId must start with 'user-' prefix"))
|
||||
|
||||
# Test serialization
|
||||
let userId = UserId(42)
|
||||
let jsonNode = %userId
|
||||
assert jsonNode.kind == JString
|
||||
assert jsonNode.getStr() == "user-42"
|
||||
|
||||
# Test deserialization
|
||||
let jsonStr = "\"user-42\""
|
||||
let parsedJson = !JsonNode.parse(jsonStr)
|
||||
let result = UserId.fromJson(parsedJson)
|
||||
assert result.isSuccess
|
||||
assert int(!result) == 42
|
||||
|
||||
# Test in object context
|
||||
type User {.serialize(mode = OptOut).} = object
|
||||
id: UserId
|
||||
name: string
|
||||
|
||||
let user = User(id: UserId(123), name: "John")
|
||||
let userJson = %user
|
||||
assert userJson.kind == JObject
|
||||
assert userJson["id"].getStr() == "user-123"
|
||||
assert userJson["name"].getStr() == "John"
|
||||
|
||||
# Test deserialization of object with custom type
|
||||
let userJsonStr = """{"id":"user-123","name":"John"}"""
|
||||
let userResult = User.fromJson(userJsonStr)
|
||||
assert userResult.isSuccess
|
||||
assert int((!userResult).id) == 123
|
||||
assert (!userResult).name == "John"
|
||||
```
|
||||
|
||||
## Implementation Details
|
||||
|
||||
The JSON serialization in nim-serde is based on the `%` operator pattern:
|
||||
|
||||
1. The `%` operator converts values to `JsonNode` objects
|
||||
2. Various overloads handle different types (primitives, objects, collections)
|
||||
3. The `toJson` function converts the `JsonNode` to a string
|
||||
|
||||
This approach makes it easy to extend with custom types by defining your own `%` operator overloads.
|
||||
|
||||
For deserialization, the library uses compile-time reflection to map JSON fields to object fields, respecting the configuration provided by pragmas.
|
||||
Loading…
x
Reference in New Issue
Block a user