More comprehensive APIs; Tests

serialization.nim

@@ -1,16 +1,76 @@
 import
-  serialization/[streams, object_serialization]
+  faststreams, serialization/object_serialization
 
 export
-  streams, object_serialization
+  faststreams, object_serialization
 
-proc encodeImpl(w: var auto, value: auto): auto =
+template serializationFormatImpl(Name: untyped,
+                                 Reader, Writer, PreferedOutput: distinct type,
+                                 mimeTypeName: static string = "") {.dirty.} =
+  # This indirection is required in order to be able to generate the
+  # `mimeType` accessor template. Without the indirection, the template
+  # mechanism of Nim will try to expand the `mimeType` param in the position
+  # of the `mimeType` template name which will result in error.
+  type Name* = object
+  template ReaderType*(T: type Name): type = Reader
+  template WriterType*(T: type Name): type = Writer
+  template PreferedOutputType*(T: type Name): type = PreferedOutput
+  template mimeType*(T: type Name): string = mimeTypeName
+
+template serializationFormat*(Name: untyped,
+                              Reader, Writer, PreferedOutput: distinct type,
+                              mimeType: static string = "") =
+  serializationFormatImpl(Name, Reader, Writer, PreferedOutput, mimeType)
+
+proc encodeImpl(writer: var auto, value: auto) =
   mixin writeValue, getOutput
-  w.writeValue value
+  writer.writeValue value
-  return w.getOutput
 
-template encode*(Writer: type, value: auto, params: varargs[untyped]): auto =
+template encode*(Format: type, value: auto, params: varargs[untyped]): auto =
-  mixin init, writeValue, getOutput
+  mixin init, WriterType, PreferedOutputType # , writeValue, getOutput
-  var w = Writer.init(params)
+  var s = init MemoryOutputStream[PreferedOutputType(Format)]
-  encodeImpl(w, value)
+
+  # TODO:
+  # Remove this when statement once the following bug is fixed:
+  # https://github.com/nim-lang/Nim/issues/9996
+  when astToStr(params) != "":
+    var writer = init(WriterType(Format), addr s, params)
+  else:
+    var writer = init(WriterType(Format), addr s)
+
+  encodeImpl(writer, value)
+  s.getOutput
+
+proc readValue*(reader: var auto, T: type): T =
+  mixin readValue
+  reader.readValue(result)
+
+proc readValueFromStream(Format: distinct type,
+                         stream: ByteStream,
+                         RecordType: distinct type): RecordType =
+  mixin init, ReaderType
+  var reader = init(ReaderType(Format), stream)
+  reader.readValue(RecordType)
+
+template decode*(Format: distinct type,
+                 input: openarray[byte] | string,
+                 RecordType: distinct type): auto =
+  var stream = memoryStream(input)
+  readValueFromStream(Format, stream, RecordType)
+
+template loadFile*(Format: distinct type,
+                   filename: string,
+                   RecordType: distinct type): auto =
+  var stream = openFile(filename)
+  readValueFromStream(Format, stream, RecordType)
+
+template loadFile*[RecordType](Format: type,
+                               filename: string,
+                               record: var RecordType) =
+  var stream = openFile(filename)
+  record = readValueFromStream(Format, stream, type(record))
+
+template saveFile*(Format: type, filename: string, args: varargs[untyped]) =
+  # TODO: This should use a proper output stream, instead of calling `encode`
+  writeFile(filename, Format.encode(args))
 

serialization.nimble

@@ -7,4 +7,6 @@ description   = "A modern and extensible serialization framework for Nim"
 license       = "Apache License 2.0"
 skipDirs      = @["tests"]
 
-requires "nim >= 0.19.0"
+requires "nim >= 0.19.0",
+         "faststreams",
+         "std_shims"

serialization/object_serialization.nim

@@ -1,15 +1,19 @@
-import macros
+import
+  std_shims/macros_shim
 
 template dontSerialize* {.pragma.}
   ## Specifies that a certain field should be ignored for
   ## the purposes of serialization
 
-template customSerialization* {.pragma.}
+type
-  ## This pragma can be applied to a record field to enable the
+  FieldMarkerImpl*[name: static string] = object
-  ## use of custom `readValue` overloads that also take a reference
+
-  ## to the object holding the field.
+  FieldReader*[RecordType, Reader] = tuple[
-  ##
+    fieldName: string,
-  ## TODO: deprecate this in favor of readField(T, field, InputArchive)
+    reader: proc (rec: var RecordType, reader: var Reader) {.nimcall.}
+  ]
+
+  FieldReadersTable*[RecordType, Reader] = openarray[FieldReader[RecordType, Reader]]
 
 template eachSerializedFieldImpl*[T](x: T, op: untyped) =
   when false:
@@ -53,3 +57,48 @@ template serializeFields*(value: auto, fieldName, fieldValue, body: untyped) =
   template op(fieldName, fieldValue: untyped) = body
   eachSerializedFieldImpl(value, op)
 
+macro customSerialization*(field: untyped, definition): untyped =
+  discard
+
+proc hasDontSerialize(pragmas: NimNode): bool =
+  if pragmas == nil: return false
+  let dontSerialize = bindSym "dontSerialize"
+  for p in pragmas:
+    if p == dontSerialize:
+      return true
+
+macro makeFieldReadersTable(RecordType, Reader: distinct type): untyped =
+  var obj = RecordType.getType[1].getImpl
+
+  result = newTree(nnkBracket)
+
+  for field in recordFields(obj):
+    let fieldName = field.name
+    if not hasDontSerialize(field.pragmas):
+      var handler = quote do:
+        return proc (obj: var `RecordType`, reader: var `Reader`) {.nimcall.} =
+          reader.readValue(obj.`fieldName`)
+
+      result.add newTree(nnkTupleConstr, newLit($fieldName), handler[0])
+
+proc fieldReadersTable*(RecordType, Reader: distinct type):
+                        ptr seq[FieldReader[RecordType, Reader]] {.gcsafe.} =
+  mixin readValue
+  var tbl {.global.} = @(makeFieldReadersTable(RecordType, Reader))
+  {.gcsafe.}:
+    return addr(tbl)
+
+proc findFieldReader*(fieldsTable: FieldReadersTable,
+                      fieldName: string,
+                      expectedFieldPos: var int): auto =
+  for i in expectedFieldPos ..< fieldsTable.len:
+    if fieldsTable[i].fieldName == fieldName:
+      expectedFieldPos = i + 1
+      return fieldsTable[i].reader
+
+  for i in 0 ..< expectedFieldPos:
+    if fieldsTable[i].fieldName == fieldName:
+      return fieldsTable[i].reader
+
+  return nil
+

serialization/streams.nim

@@ -1,86 +0,0 @@
-# This module will be overhauled in the future to use concepts
-
-type
-  MemoryStream* = object
-    output: seq[byte]
-
-  StringStream* = object
-    output: string
-
-  AnyStream = MemoryStream | StringStream
-
-const
-  initialStreamCapacity = 4096
-
-# Memory stream
-
-proc init*(s: var MemoryStream) =
-  s.output = newSeqOfCap[byte](initialStreamCapacity)
-
-proc getOutput*(s: var MemoryStream): seq[byte] =
-  shallow s.output
-  result = s.output
-
-proc init*(T: type MemoryStream): T =
-  init result
-
-proc append*(s: var MemoryStream, b: byte) =
-  s.output.add b
-
-proc append*(s: var MemoryStream, c: char) =
-  s.output.add byte(c)
-
-proc append*(s: var MemoryStream, bytes: openarray[byte]) =
-  s.output.add bytes
-
-proc append*(s: var MemoryStream, chars: openarray[char]) =
-  # TODO: this can be optimized
-  for c in chars:
-    s.output.add byte(c)
-
-template append*(s: var MemoryStream, str: string) =
-  s.append(str.toOpenArrayByte(0, str.len - 1))
-
-# String stream
-
-proc init*(s: var StringStream) =
-  s.output = newStringOfCap(initialStreamCapacity)
-
-proc getOutput*(s: var StringStream): string =
-  shallow s.output
-  result = s.output
-
-proc init*(T: type StringStream): T =
-  init result
-
-proc append*(s: var StringStream, c: char) =
-  s.output.add c
-
-proc append*(s: var StringStream, chars: openarray[char]) =
-  # TODO: Nim doesn't have add(openarray[char]) for strings
-  for c in chars:
-    s.output.add c
-
-template append*(s: var StringStream, str: string) =
-  s.output.add str
-
-# Any stream
-
-proc appendNumberImpl(s: var AnyStream, number: BiggestInt) =
-  # TODO: don't allocate
-  s.append $number
-
-proc appendNumberImpl(s: var AnyStream, number: BiggestUInt) =
-  # TODO: don't allocate
-  s.append $number
-
-template toBiggestRepr(i: SomeUnsignedInt): BiggestUInt =
-  BiggestUInt(i)
-
-template toBiggestRepr(i: SomeSignedInt): BiggestInt =
-  BiggestInt(i)
-
-template appendNumber*(s: var AnyStream, i: SomeInteger) =
-  # TODO: specify radix/base
-  appendNumberImpl(s, toBiggestRepr(i))
-

serialization/testing/generic_suite.nim

@@ -0,0 +1,105 @@
+import
+  unittest, times, typetraits,
+  faststreams/input_stream,
+  ../object_serialization
+
+type
+  Transaction = object
+    amount: int
+    time: DateTime
+    sender: string
+    receiver: string
+
+  Foo = object
+    x: uint64
+    y: string
+    z: seq[int]
+
+  Bar = object
+    b: string
+    f: Foo
+
+  # Baz should use custom serialization
+  # The `i` field should be multiplied by two while deserialing and
+  # `ignored` field should be set to 10
+  Baz = object
+    f: Foo
+    i: int
+    ignored {.dontSerialize.}: int
+
+proc default(T: typedesc): T = discard
+
+proc executeReaderWriterTests*(Format: type) =
+  mixin init, ReaderType, WriterType
+
+  type
+    Reader = ReaderType Format
+    Writer = WriterType Format
+
+  suite(typetraits.name(Format) & " read/write tests"):
+    test "Low-level field reader test":
+      let barFields = fieldReadersTable(Bar, Reader)
+      var idx = 0
+
+      var fieldReader = findFieldReader(barFields[], "b", idx)
+      check fieldReader != nil and idx == 1
+
+      # check that the reader can be found again starting from a higher index
+      fieldReader = findFieldReader(barFields[], "b", idx)
+      check fieldReader != nil and idx == 1
+
+      var bytes = Format.encode("test")
+      var stream = memoryStream(bytes)
+      var reader = Reader.init(stream)
+
+      var bar: Bar
+      fieldReader(bar, reader)
+
+      check bar.b == "test"
+
+    test "Ignored fields should not be included in the field readers table":
+      var pos = 0
+      let bazFields = fieldReadersTable(Baz, Reader)
+      check:
+        len(bazFields[]) == 2
+        findFieldReader(bazFields[], "f", pos) != nil
+        findFieldReader(bazFields[], "i", pos) != nil
+        findFieldReader(bazFields[], "i", pos) != nil
+        findFieldReader(bazFields[], "f", pos) != nil
+        findFieldReader(bazFields[], "f", pos) != nil
+        findFieldReader(bazFields[], "ignored", pos) == nil
+        findFieldReader(bazFields[], "some_other_name", pos) == nil
+
+    test "Encoding and decoding an object":
+      var originalBar = Bar(b: "abracadabra",
+                            f: Foo(x: 5'u64, y: "hocus pocus", z: @[100, 200, 300]))
+
+      var bytes = Format.encode(originalBar)
+      var s = memoryStream(bytes)
+      var reader = Reader.init(s)
+      var restoredBar = reader.readValue(Bar)
+
+      check:
+        originalBar == restoredBar
+
+      when false:
+        var t1 = Transaction(time: now(), amount: 1000, sender: "Alice", receiver: "Bob")
+        bytes = Format.encode(t1)
+        var t2 = Format.decode(bytes, Transaction)
+
+        check:
+          t2.time == default(DateTime)
+          t2.sender == "Alice"
+          t2.receiver == "Bob"
+          t2.amount == 1000
+
+        var origVal = Baz(f: Foo(x: 10'u64, y: "y", z: @[]), ignored: 5)
+        bytes = Format.encode(origVal)
+        var restored = Format.decode(bytes, Baz)
+
+        check:
+          origVal.f.x == restored.f.x
+          origVal.f.i == restored.f.i div 2
+          origVal.f.y.len == restored.f.y.len
+          restored.ignored == 10
+