status-im
/
nim-protobuf-serialization
mirror of https://github.com/status-im/nim-protobuf-serialization.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Initial version of the library completed (#2) 

Initial version of the library integrated with nim-serialization

Co-authored-by: Luke Parker <lukeparker5132@gmail.com>
This commit is contained in:
zah 2020-07-14 15:51:44 +03:00 committed by GitHub
parent 879421b121
commit b5fd5611ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
47 changed files with 2529 additions and 661 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
nimcache/

408
protobuf_serialization.nim
View File

@ -1,376 +1,32 @@
import macros, strformat, typetraits, options
import faststreams
template sint32*() {.pragma.}
template sint64*() {.pragma.}
template sfixed32*() {.pragma.}
template sfixed64*() {.pragma.}
template fixed32*() {.pragma.}
template fixed64*() {.pragma.}
template float*() {.pragma.}
template double*() {.pragma.}
const
MaxMessageSize* = 1'u shl 22
type
ProtoBuffer* = object
fieldNum: int
outstream: OutputStreamVar
ProtoWireType* = enum
## Protobuf's field types enum
Varint, Fixed64, LengthDelimited, StartGroup, EndGroup, Fixed32
EncodingKind* = enum
ekNormal, ekZigzag
ProtoField*[T] = object
## Protobuf's message field representation object
index*: int
value*: T
SomeSVarint* = int | int64 | int32 | int16 | int8 | enum
SomeByte* = byte | bool | char | uint8
SomeUVarint* = uint | uint64 | uint32 | uint16 | SomeByte
SomeVarint* = SomeSVarint | SomeUVarint
SomeLengthDelimited* = string | seq[SomeByte] | cstring
SomeFixed64* = float64
SomeFixed32* = float32
SomeFixed* = SomeFixed32 | SomeFixed64
AnyProtoType* = SomeVarint | SomeLengthDelimited | SomeFixed | object
UnexpectedTypeError* = object of ValueError
proc newProtoBuffer*(): ProtoBuffer =
ProtoBuffer(outstream: OutputStream.init(), fieldNum: 1)
proc output*(proto: ProtoBuffer): seq[byte] {.inline.} =
proto.outstream.getOutput
template wireType(firstByte: byte): ProtoWireType =
(firstByte and 0b111).ProtoWireType
template fieldNumber(firstByte: byte): int =
((firstByte shr 3) and 0b1111).int
template protoHeader*(fieldNum: int, wire: ProtoWireType): byte =
## Get protobuf's field header integer for ``index`` and ``wire``.
((cast[uint](fieldNum) shl 3) or cast[uint](wire)).byte
template increaseBytesRead(amount = 1) =
## Convenience template for increasing
## all of the counts
mixin isSome
bytesRead += amount
outOffset += amount
outBytesProcessed += amount
if numBytesToRead.isSome():
if (bytesRead > numBytesToRead.get()).unlikely:
raise newException(Exception, &"Number of bytes read ({bytesRead}) exceeded bytes requested ({numBytesToRead})")
proc encodeField*[T: not AnyProtoType](protobuf: var ProtoBuffer, value: T) {.inline.}
proc encodeField*[T: not AnyProtoType](protobuf: var ProtoBuffer, fieldNum: int, value: T) {.inline.}
proc encodeField[T: not AnyProtoType](stream: OutputStreamVar, fieldNum: int, value: T) {.inline.}
proc put(stream: OutputStreamVar, value: SomeVarint) {.inline.} =
when value is enum:
var value = cast[type(ord(value))](value)
elif value is bool or value is char:
var value = cast[byte](value)
else:
var value = value
when type(value) is SomeSVarint:
# Encode using zigzag
if value < type(value)(0):
value = not(value shl type(value)(1))
else:
value = value shl type(value)(1)
while value > type(value)(0b0111_1111):
stream.append byte((value and 0b0111_1111) or 0b1000_0000)
value = value shr 7
stream.append byte(value and 0b1111_1111)
proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeVarint) {.inline.} =
stream.append protoHeader(fieldNum, Varint)
stream.put(value)
proc put(stream: OutputStreamVar, value: SomeFixed) {.inline.} =
when typeof(value) is SomeFixed64:
var value = cast[int64](value)
else:
var value = cast[int32](value)
for _ in 0 ..< sizeof(value):
stream.append byte(value and 0b1111_1111)
value = value shr 8
proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeFixed64) {.inline.} =
stream.append protoHeader(fieldNum, Fixed64)
stream.put(value)
proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeFixed32) {.inline.} =
stream.append protoHeader(fieldNum, Fixed32)
stream.put(value)
proc put(stream: OutputStreamVar, value: SomeLengthDelimited) {.inline.} =
stream.put(len(value).uint)
for b in value:
stream.append byte(b)
proc encodeField(stream: OutputStreamVar, fieldNum: int, value: SomeLengthDelimited) {.inline.} =
stream.append protoHeader(fieldNum, LengthDelimited)
stream.put(value)
proc put(stream: OutputStreamVar, value: object) {.inline.}
proc encodeField(stream: OutputStreamVar, fieldNum: int, value: object) {.inline.} =
# This is currently needed in order to get the size
# of the output before adding it to the stream.
# Maybe there is a better way to do this
let objStream = OutputStream.init()
objStream.put(value)
let objOutput = objStream.getOutput()
if objOutput.len > 0:
stream.append protoHeader(fieldNum, LengthDelimited)
stream.put(objOutput)
proc put(stream: OutputStreamVar, value: object) {.inline.} =
var fieldNum = 1
for _, val in value.fieldPairs:
# Only store the value
if default(type(val)) != val:
stream.encodeField(fieldNum, val)
inc fieldNum
proc encode*(protobuf: var ProtoBuffer, value: object) {.inline.} =
protobuf.outstream.put(value)
proc encodeField*(protobuf: var ProtoBuffer, fieldNum: int, value: AnyProtoType) {.inline.} =
protobuf.outstream.encodeField(fieldNum, value)
proc encodeField*(protobuf: var ProtoBuffer, value: AnyProtoType) {.inline.} =
protobuf.encodeField(protobuf.fieldNum, value)
inc protobuf.fieldNum
proc encodeField[T: not AnyProtoType](stream: OutputStreamVar, fieldNum: int, value: T) {.inline.} =
stream.encodeField(fieldNum, value.toBytes)
proc encodeField*[T: not AnyProtoType](protobuf: var ProtoBuffer, fieldNum: int, value: T) {.inline.} =
protobuf.outstream.encodeField(fieldNum, value.toBytes)
proc encodeField*[T: not AnyProtoType](protobuf: var ProtoBuffer, value: T) {.inline.} =
protobuf.encodeField(protobuf.fieldNum, value.toBytes)
inc protobuf.fieldNum
proc get*[T: SomeFixed](
bytes: var seq[byte],
ty: typedesc[T],
outOffset: var int,
outBytesProcessed: var int,
numBytesToRead = none(int)
): T {.inline.} =
var bytesRead = 0
when T is SomeFixed64:
var value: int64
else:
var value: int32
var shiftAmount = 0
for _ in 0 ..< sizeof(T):
value += type(value)(bytes[outOffset]) shl shiftAmount
shiftAmount += 8
increaseBytesRead()
result = cast[T](value)
proc get[T: SomeVarint](
bytes: var seq[byte],
ty: typedesc[T],
outOffset: var int,
outBytesProcessed: var int,
numBytesToRead = none(int)
): T {.inline.} =
var bytesRead = 0
# Only up to 128 bits supported by the spec
when T is enum or T is char:
var value: type(ord(result))
elif T is bool:
var value: byte
else:
var value: T
var shiftAmount = 0
while true:
value += type(value)(bytes[outOffset] and 0b0111_1111) shl shiftAmount
shiftAmount += 7
if (bytes[outOffset] shr 7) == 0:
break
increaseBytesRead()
increaseBytesRead()
when ty is SomeSVarint:
if (value and type(value)(1)) != type(value)(0):
result = cast[T](not(value shr type(value)(1)))
else:
result = cast[T](value shr type(value)(1))
else:
result = T(value)
proc checkType[T: SomeVarint](tyByte: byte, ty: typedesc[T], offset: int) {.inline.} =
let wireTy = wireType(tyByte)
if wireTy != Varint:
raise newException(UnexpectedTypeError, fmt"Not a varint at offset {offset}! Received a {wireTy}")
proc checkType[T: SomeFixed](tyByte: byte, ty: typedesc[T], offset: int) {.inline.} =
let wireTy = wireType(tyByte)
if wireTy notin {Fixed32, Fixed64}:
raise newException(UnexpectedTypeError, fmt"Not a fixed32 or fixed64 at offset {offset}! Received a {wireTy}")
proc checkType[T: SomeLengthDelimited](tyByte: byte, ty: typedesc[T], offset: int) {.inline.} =
let wireTy = wireType(tyByte)
if wireTy != LengthDelimited:
raise newException(UnexpectedTypeError, fmt"Not a length delimited value at offset {offset}! Received a {wireTy}")
proc checkType[T: object](tyByte: byte, ty: typedesc[T], offset: int) {.inline.} =
let wireTy = wireType(tyByte)
if wireTy != LengthDelimited:
raise newException(UnexpectedTypeError, fmt"Not an object value at offset {offset}! Received a {wireTy}")
proc get*[T: SomeLengthDelimited](
bytes: var seq[byte],
ty: typedesc[T],
outOffset: var int,
outBytesProcessed: var int,
numBytesToRead = none(int)
): T {.inline.} =
var bytesRead = 0
let decodedSize = bytes.get(uint, outOffset, outBytesProcessed, numBytesToRead)
let length = decodedSize.int
when T is string:
result = newString(length)
for i in outOffset ..< (outOffset + length):
result[i - outOffset] = bytes[i].chr
elif T is cstring:
result = cast[cstring](bytes[outOffset ..< (outOffset + length)])
else:
result.setLen(length)
for i in outOffset ..< (outOffset + length):
result[i - outOffset] = type(result[0])(bytes[i])
increaseBytesRead(length)
proc decodeField*[T: SomeFixed | SomeVarint | SomeLengthDelimited](
bytes: var seq[byte],
ty: typedesc[T],
outOffset: var int,
outBytesProcessed: var int,
numBytesToRead = none(int)
): ProtoField[T] {.inline.} =
var bytesRead = 0
checkType(bytes[outOffset], ty, outOffset)
result.index = fieldNumber(bytes[outOffset])
increaseBytesRead()
result.value = bytes.get(ty, outOffset, outBytesProcessed, numBytesToRead)
proc decodeField*[T: object](
bytes: var seq[byte],
ty: typedesc[T],
outOffset: var int,
outBytesProcessed: var int,
numBytesToRead = none(int)
): ProtoField[T] {.inline.}
proc decodeField*[T: not AnyProtoType](
bytes: var seq[byte],
ty: typedesc[T],
outOffset: var int,
outBytesProcessed: var int,
numBytesToRead = none(int)
): ProtoField[T] {.inline.} =
var bytesRead = 0
checkType(bytes[outOffset], seq[byte], outOffset)
result.index = fieldNumber(bytes[outOffset])
increaseBytesRead()
var value = bytes.get(seq[byte], outOffset, outBytesProcessed, numBytesToRead)
result.value = value.to(T)
macro setField(obj: typed, fieldNum: int, offset: int, bytesProcessed: int, bytesToRead: Option[int], value: untyped): untyped =
let typeFields = obj.getTypeInst.getType
let objFields = typeFields[2]
expectKind objFields, nnkRecList
result = newStmtList()
let caseStmt = newNimNode(nnkCaseStmt)
caseStmt.add(fieldNum)
for i in 0 ..< len(objFields) - 1:
let field = objFields[i]
let ofBranch = newNimNode(nnkOfBranch)
ofBranch.add(newLit(i+1))
ofBranch.add(
quote do:
`obj`.`field` = decodeField(`value`, type(`obj`.`field`), `offset`, `bytesProcessed`, `bytesToRead`).value
)
caseStmt.add(ofBranch)
let field = objFields[len(objFields) - 1]
let elseBranch = newNimNode(nnkElse)
elseBranch.add(
nnkStmtList.newTree(
quote do:
`obj`.`field` = decodeField(`value`, type(`obj`.`field`), `offset`, `bytesProcessed`, `bytesToRead`).value
)
)
caseStmt.add(elseBranch)
result.add(caseStmt)
proc decodeField*[T: object](
bytes: var seq[byte],
ty: typedesc[T],
outOffset: var int,
outBytesProcessed: var int,
numBytesToRead = none(int)
): ProtoField[T] {.inline.} =
var bytesRead = 0
checkType(bytes[outOffset], ty, outOffset)
result.index = fieldNumber(bytes[outOffset])
# read LD header
# then read only amount of bytes needed
increaseBytesRead()
let decodedSize = bytes.get(uint, outOffset, outBytesProcessed, numBytesToRead)
let bytesToRead = some(decodedSize.int)
let oldOffset = outOffset
while outOffset < oldOffset + bytesToRead.get():
let fieldNum = fieldNumber(bytes[outOffset])
setField(result.value, fieldNum, outOffset, outBytesProcessed, bytesToRead, bytes)
proc decode*[T: object](
bytes: var seq[byte],
ty: typedesc[T],
): T {.inline.} =
var bytesRead = 0
var offset = 0
while offset < bytes.len - 1:
let fieldNum = fieldNumber(bytes[offset])
setField(result, fieldNum, offset, bytesRead, none(int), bytes)
import sets
import serialization
export serialization
import protobuf_serialization/[internal, types, reader, writer]
export types, reader, writer
serializationFormat Protobuf,
Reader = ProtobufReader,
Writer = ProtobufWriter,
PreferedOutput = seq[byte]
func supportsInternal[T](ty: typedesc[T], handled: var HashSet[string]) {.compileTime.} =
if handled.contains($T):
return
handled.incl($T)
verifySerializable(T)
var inst: T
enumInstanceSerializedFields(inst, fieldName, fieldVar):
discard fieldName
when flatType(fieldVar) is (object or tuple):
supportsInternal(flatType(fieldVar), handled)
func supportsCompileTime[T](_: typedesc[T]) =
var handled = initHashSet[string]()
when flatType(T) is (object or tuple):
supportsInternal(flatType(T), handled)
func supports*[T](_: type Protobuf, ty: typedesc[T]): bool =
static: supportsCompileTime(T)

26
protobuf_serialization.nimble
View File

@ -1,14 +1,32 @@
# Package
import os
mode = ScriptMode.Verbose
version = "0.1.0"
author = "Joey Yakimowich-Payne"
description = "Protobuf implementation compatible with the nim-serialization framework."
license = "MIT"
srcDir = "src"
skipDirs = @["tests"]
requires "nim >= 1.2.0",
"stew",
"faststreams",
"serialization"
task test, "Run all tests":
#Explicitly specify the call depth limit in case the default changes in the future.
exec "nim c -r --threads:off tests/test_all"
exec "nim c -r --threads:on tests/test_all"
# Dependencies
#Also iterate over every test in tests/fail, and verify they fail to compile.
echo "\r\n\x1B[0;94m[Suite]\x1B[0;37m Test Fail to Compile"
var tests: seq[string] = @[]
for path in listFiles(thisDir() / "tests" / "fail"):
if path.split(".")[^1] != "nim":
continue
requires "nim >= 1.0.6", "faststreams"
if gorgeEx("nim c " & path).exitCode != 0:
echo " \x1B[0;92m[OK]\x1B[0;37m ", path.split("/")[^1]
else:
echo " \x1B[0;31m[FAILED]\x1B[0;37m ", path.split("/")[^1]
exec "exit 1"

275
protobuf_serialization/internal.nim Normal file
View File

@ -0,0 +1,275 @@
#Variables needed by the Reader and Writer which should NOT be exported outside of this library.
import options
import sets
import tables
import stew/shims/macros
#Depending on the situation, one of these two are used.
#Sometimes, one works where the other doesn't.
#It all comes down to bugs in Nim and managing them.
export getCustomPragmaVal, getCustomPragmaFixed
export hasCustomPragmaFixed
import serialization
import numbers/varint
export varint
import numbers/fixed
export fixed
const WIRE_TYPE_MASK = 0b0000_0111'i32
type
ProtobufWireType* = enum
VarInt, Fixed64, LengthDelimited, StartGroup, EndGroup, Fixed32
ProtobufKey* = object
number*: int
wire*: ProtobufWireType
#Number types which are platform-dependent and therefore unsafe.
PlatformDependentTypes* = int or uint
#Castable length delimited types.
#These can be directly casted from a seq[byte] and do not require a custom converter.
CastableLengthDelimitedTypes* = seq[byte or char or bool]
#This type is literally every other type.
#Every other type is considered custom, due to the need for their own converters.
#While cstring/array are built-ins, and therefore should have converters provided, but they still need converters.
LengthDelimitedTypes* = not (VarIntTypes or FixedTypes)
#Disabled types.
Disabled = LUIntWrapped or array or cstring or tuple or Table
const DISABLED_STRING = "Arrays, cstrings, tuples, and Tables are not serializable due to various reasons."
discard DISABLED_STRING
template isPotentiallyNull*[T](ty: typedesc[T]): bool =
T is (Option or ref or ptr)
template getUnderlyingType*[I](
stdlib: seq[I] or set[I] or HashSet[I]
): untyped =
I
proc flatTypeInternal(value: auto): auto {.compileTime.} =
when value is Option:
flatTypeInternal(value.get())
elif value is (ref or ptr):
flatTypeInternal(value[])
else:
value
template flatType*(value: auto): type =
type(flatTypeInternal(value))
template flatType*[B](ty: typedesc[B]): type =
when B is openArray:
B
else:
var blank: B
flatType(blank)
proc flatMapInternal[B, T](value: B, ty: typedesc[T]): Option[T] =
when value is Option:
if value.isNone():
return
flatMapInternal(value.get(), ty)
elif value is (ref or ptr):
if value.isNil():
return
flatMapInternal(value[], ty)
else:
some(value)
template flatMap*(value: auto): auto =
flatMapInternal(value, flatType(value))
func isStdlib*[B](_: typedesc[B]): bool {.compileTime.} =
flatType(B) is (cstring or string or seq or array or set or HashSet)
func mustUseSingleBuffer*[T](_: typedesc[T]): bool {.compileTime.}
func convertAndCallMustUseSingleBuffer[T](
_: typedesc[seq[T] or openArray[T] or set[T] or HashSet[T]]
): bool {.compileTime.} =
when flatType(T).isStdlib():
false
else:
mustUseSingleBuffer(flatType(T))
#[func convertAndCallMustUseSingleBuffer[C, T](
_: typedesc[array[C, T]]
): bool {.compileTime.} =
when flatType(T).isStdlib():
false
else:
mustUseSingleBuffer(flatType(T))]#
func mustUseSingleBuffer*[T](_: typedesc[T]): bool {.compileTime.} =
when flatType(T) is (cstring or string or seq[byte or char or bool]):
true
elif flatType(T) is (array or openArray or set or HashSet):
flatType(T).convertAndCallMustUseSingleBuffer()
else:
false
func singleBufferable*[T](_: typedesc[T]): bool {.compileTime.}
func convertAndCallSingleBufferable[T](
_: typedesc[seq[T] or openArray[T] or set[T] or HashSet[T]]
): bool {.compileTime.} =
when flatType(T).isStdlib():
false
else:
singleBufferable(flatType(T))
#[func convertAndCallSingleBufferable[C, T](
_: typedesc[array[C, T]]
): bool {.compileTime.} =
when flatType(T).isStdlib():
false
else:
singleBufferable(flatType(T))]#
func singleBufferable*[T](_: typedesc[T]): bool {.compileTime.} =
when flatType(T).mustUseSingleBuffer():
true
elif flatType(T) is (VarIntTypes or FixedTypes):
true
elif flatType(T) is (seq or array or openArray or set or HashSet):
flatType(T).convertAndCallSingleBufferable()
else:
false
template nextType[B](box: B): auto =
when B is Option:
box.get()
elif B is (ref or ptr):
box[]
else:
box
proc boxInternal[C, B](value: C, into: B): B =
when value is B:
value
elif into is Option:
var blank: type(nextType(into))
#We never access this pointer.
#Ever.
#That said, in order for this Option to resolve as some, it can't be nil.
when blank is ref:
blank = cast[type(blank)](1)
elif blank is ptr:
blank = cast[type(blank)](1)
let temp = some(blank)
some(boxInternal(value, nextType(temp)))
elif into is ref:
new(result)
result[] = boxInternal(value, nextType(result))
elif into is ptr:
result = cast[B](alloc0(sizeof(B)))
result[] = boxInternal(value, nextType(result))
proc box*[B](into: var B, value: auto) =
into = boxInternal(value, into)
template fieldNumber*(num: int) {.pragma.}
template dontOmit*() {.pragma.}
#Created in response to https://github.com/kayabaNerve/nim-protobuf-serialization/issues/5.
func verifySerializable*[T](ty: typedesc[T]) {.compileTime.} =
when T is PlatformDependentTypes:
{.fatal: "Serializing a number requires specifying the amount of bits via the type.".}
elif T is SomeFloat:
{.fatal: "Couldnt serialize the float; all floats need their bits specified with a PFloat32 or PFloat64 call.".}
elif T is PureTypes:
{.fatal: "Serializing a number requires specifying the encoding to use.".}
#LUIntWrapped is disabled; provide a better error.
elif T is LUIntWrapped:
{.fatal: "LibP2P VarInts are only usable directly with encodeVarInt.".}
elif T is Disabled:
{.fatal: DISABLED_STRING & " are not serializable due to various reasons.".}
elif T.isStdlib():
discard
#Tuple inclusion is so in case we can add back support for tuples, we solely have to delete the above case.
elif T is (object or tuple):
var
inst: T
fieldNumberSet = initHashSet[int]()
discard fieldNumberSet
enumInstanceSerializedFields(inst, fieldName, fieldVar):
discard fieldName
when fieldVar is PlatformDependentTypes:
{.fatal: "Serializing a number requires specifying the amount of bits via the type.".}
elif T is LUIntWrapped:
{.fatal: "LibP2P VarInts are only usable directly with encodeVarInt.".}
elif T is Disabled:
{.fatal: DISABLED_STRING & " are not serializable due to various reasons.".}
elif fieldVar is (VarIntTypes or FixedTypes):
const
hasPInt = ty.hasCustomPragmaFixed(fieldName, pint)
hasSInt = ty.hasCustomPragmaFixed(fieldName, sint)
hasFixed = ty.hasCustomPragmaFixed(fieldName, fixed)
when fieldVar is (VarIntWrapped or FixedWrapped):
when uint(hasPInt) + uint(hasSInt) + uint(hasFixed) != 0:
{.fatal: "Encoding specified for an already wrapped type, or a type which isn't wrappable due to always having one encoding (byte, char, bool, or float).".}
when fieldVar is SomeFloat:
const
hasF32 = ty.hasCustomPragmaFixed(fieldName, pfloat32)
hasF64 = ty.hasCustomPragmaFixed(fieldName, pfloat64)
when hasF32:
when sizeof(fieldVar) != 4:
{.fatal: "pfloat32 pragma attached to a 64-bit float.".}
elif hasF64:
when sizeof(fieldVar) != 8:
{.fatal: "pfloat64 pragma attached to a 32-bit float.".}
else:
{.fatal: "Floats require the pfloat32 or pfloat64 pragma to be attached.".}
elif uint(hasPInt) + uint(hasSInt) + uint(hasFixed) != 1:
{.fatal: "Couldn't write " & fieldName & "; either none or multiple encodings were specified.".}
const thisFieldNumber = fieldVar.getCustomPragmaVal(fieldNumber)
when thisFieldNumber is NimNode:
{.fatal: "No field number specified on serialized field.".}
else:
when thisFieldNumber <= 0:
{.fatal: "Negative field number or 0 field number was specified. Protobuf fields start at 1.".}
elif thisFieldNumber shr 28 != 0:
#I mean, it is technically serializable with an uint64 (max 2^60), or even uint32 (max 2^29).
#That said, having more than 2^28 fields should never be needed. Why lose performance for a never-useful case?
{.fatal: "Field number greater than 2^28 specified. On 32-bit systems, this isn't serializable.".}
if fieldNumberSet.contains(thisFieldNumber):
raise newException(Exception, "Field number was used twice on two different fields.")
fieldNumberSet.incl(thisFieldNumber)
proc newProtobufKey*(number: int, wire: ProtobufWireType): seq[byte] =
result = newSeq[byte](5)
var viLen = 0
doAssert encodeVarInt(
result,
viLen,
PInt((int32(number) shl 3) or int32(wire))
) == VarIntStatus.Success
result.setLen(viLen)
proc writeProtobufKey*(
stream: OutputStream,
number: int,
wire: ProtobufWireType
) {.inline.} =
stream.write(newProtobufKey(number, wire))
proc readProtobufKey*(
stream: InputStream
): ProtobufKey =
let
varint = stream.decodeVarInt(int, PInt(int32))
wire = byte(varint and WIRE_TYPE_MASK)
if (wire < byte(low(ProtobufWireType))) or (byte(high(ProtobufWireType)) < wire):
raise newException(ProtobufMessageError, "Protobuf key had an invalid wire type.")
result.wire = ProtobufWireType(wire)
result.number = varint shr 3

87
protobuf_serialization/numbers/common.nim Normal file
View File

@ -0,0 +1,87 @@
import macros
import serialization
type
#Defined here so the number encoders/decoders have access.
ProtobufError* = object of SerializationError
ProtobufReadError* = object of ProtobufError
ProtobufEOFError* = object of ProtobufReadError
ProtobufMessageError* = object of ProtobufReadError
#Signed native types.
PureSIntegerTypes* = SomeSignedInt or enum
#Unsigned native types.
PureUIntegerTypes* = SomeUnsignedInt or char or bool
#Every native type.
PureTypes* = (PureSIntegerTypes or PureUIntegerTypes) and
(not (byte or char or bool))
macro generateWrapper*(
name: untyped,
supported: typed,
exclusion: typed,
uTypes: typed,
uLarger: typed,
uSmaller: typed,
sTypes: typed,
sLarger: typed,
sSmaller: typed,
err: string
): untyped =
let strLitName = newStrLitNode(name.strVal)
quote do:
template `name`*(value: untyped): untyped =
when (value is (bool or byte or char)) and (`strLitName` != "PInt"):
{.fatal: "Byte types are always PInt.".}
#If this enum doesn't have negative values, considered it unsigned.
when value is enum:
when value is type:
when ord(low(value)) < 0:
type fauxType = int32
else:
type fauxType = uint32
else:
when ord(low(type(value))) < 0:
type fauxType = int32
else:
type fauxType = uint32
elif value is type:
type fauxType = value
else:
type fauxType = type(value)
when fauxType is not `supported`:
{.fatal: `err`.}
elif fauxType is `exclusion`:
{.fatal: "Tried to rewrap a wrapped type.".}
when value is type:
when fauxType is `uTypes`:
when sizeof(value) == 8:
`uLarger`
else:
`uSmaller`
elif fauxType is `sTypes`:
when sizeof(value) == 8:
`sLarger`
else:
`sSmaller`
#Used for Fixed floats.
else:
value
else:
when fauxType is `uTypes`:
when sizeof(value) == 8:
`uLarger`(value)
else:
`uSmaller`(value)
elif fauxType is `sTypes`:
when sizeof(value) == 8:
`sLarger`(value)
else:
`sSmaller`(value)
else:
value

85
protobuf_serialization/numbers/fixed.nim Normal file
View File

@ -0,0 +1,85 @@
import faststreams
import common
export PureTypes
export ProtobufError, ProtobufReadError, ProtobufEOFError, ProtobufMessageError
const LAST_BYTE* = 0b1111_1111
type
FixedWrapped64 = distinct uint64
FixedWrapped32 = distinct uint32
SFixedWrapped64 = distinct int64
SFixedWrapped32 = distinct int32
FloatWrapped64 = distinct uint64
FloatWrapped32 = distinct uint32
FixedDistinctWrapped = FixedWrapped64 or FixedWrapped32 or
SFixedWrapped64 or SFixedWrapped32 or
FloatWrapped64 or FloatWrapped32
FixedWrapped* = FixedDistinctWrapped or float64 or float32
WrappableFixedTypes = PureUIntegerTypes or PureSIntegerTypes
#Every type valid for the Fixed (64 or 43) wire type.
FixedTypes* = FixedWrapped or WrappableFixedTypes
generateWrapper(
Fixed, WrappableFixedTypes, FixedDistinctWrapped,
PureUIntegerTypes, FixedWrapped64, FixedWrapped32,
PureSIntegerTypes, SFixedWrapped64, SFixedWrapped32,
"Fixed should only be used with a non-float number. Floats are always fixed already."
)
template Float64*(value: float64): FloatWrapped64 =
cast[FloatWrapped64](value)
template Float32*(value: float32): FloatWrapped32 =
cast[FloatWrapped32](value)
template unwrap*(value: FixedWrapped): untyped =
when value is FixedWrapped64:
uint64(value)
elif value is FixedWrapped32:
uint32(value)
elif value is SFixedWrapped64:
int64(value)
elif value is SFixedWrapped32:
int32(value)
elif value is FloatWrapped64:
float64(value)
elif value is FloatWrapped32:
float32(value)
elif value is (float64 or float32):
value
else:
{.fatal: "Tried to get the unwrapped value of a non-wrapped type. This should never happen.".}
template fixed*() {.pragma.}
template pfloat32*() {.pragma.}
template pfloat64*() {.pragma.}
proc encodeFixed*(stream: OutputStream, value: FixedWrapped) =
when sizeof(value) == 8:
var casted = cast[uint64](value)
else:
var casted = cast[uint32](value)
for _ in 0 ..< sizeof(casted):
stream.write(byte(casted and LAST_BYTE))
casted = casted shr 8
proc decodeFixed*(
stream: InputStream,
res: var auto
) =
when sizeof(res) == 8:
var temp: uint64
else:
var temp: uint32
for i in 0 ..< sizeof(temp):
if not stream.readable():
raise newException(ProtobufEOFError, "Stream ended before the Fixed number was finished.")
temp = temp + (type(temp)(stream.read()) shl (i * 8))
res = cast[type(res)](temp)

279
protobuf_serialization/numbers/varint.nim Normal file
View File

@ -0,0 +1,279 @@
import stew/bitops2
import faststreams
import common
export PureTypes
export ProtobufError, ProtobufReadError, ProtobufEOFError, ProtobufMessageError
const
VAR_INT_CONTINUATION_MASK*: byte = 0b1000_0000
VAR_INT_VALUE_MASK: byte = 0b0111_1111
type
VarIntStatus* = enum
Success,
Overflow,
Incomplete
#Used to specify how to encode/decode primitives.
#Despite being used outside of this library, all access is via templates.
PIntWrapped32 = distinct int32
PIntWrapped64 = distinct int64
SIntWrapped32 = distinct int32
SIntWrapped64 = distinct int64
UIntWrapped32 = distinct uint32
UIntWrapped64 = distinct uint64
LUIntWrapped32 = distinct uint32
LUIntWrapped64 = distinct uint64
#Types which share an encoding.
PIntWrapped = PIntWrapped32 or PIntWrapped64
SIntWrapped = SIntWrapped32 or SIntWrapped64
UIntWrapped = UIntWrapped32 or UIntWrapped64 or
byte or char or bool
LUIntWrapped* = LUIntWrapped32 or LUIntWrapped64
#Any wrapped VarInt types.
VarIntWrapped* = PIntWrapped or SIntWrapped or
UIntWrapped or LUIntWrapped
#Every signed integer Type.
SIntegerTypes = PureSIntegerTypes or
PIntWrapped32 or PIntWrapped64 or
SIntWrapped32 or SIntWrapped64
#Every unsigned integer Type.
UIntegerTypes = PureUIntegerTypes or UIntWrapped or LUIntWrapped
#Every type valid for the VarInt wire type.
VarIntTypes* = SIntegerTypes or UIntegerTypes
generateWrapper(
PInt, UIntegerTypes or SIntegerTypes, VarIntWrapped,
UIntegerTypes, UIntWrapped64, UIntWrapped32,
SIntegerTypes, PIntWrapped64, PIntWrapped32,
"LInt should only be used with integers (signed or unsigned)."
)
generateWrapper(
SInt, SIntegerTypes, VarIntWrapped,
void, void, void,
SIntegerTypes, SIntWrapped64, SIntWrapped32,
"SInt should only be used with signed integers."
)
generateWrapper(
LInt, UIntegerTypes, VarIntWrapped,
UIntegerTypes, LUIntWrapped64, LUIntWrapped32,
void, void, void,
"LInt should only be used with unsigned integers."
)
#Used to specify how to encode/decode fields in an object.
template pint*() {.pragma.}
template sint*() {.pragma.}
template unwrap*(value: VarIntWrapped): untyped =
when value is (PIntWrapped32 or SIntWrapped32):
int32(value)
elif value is (PIntWrapped64 or SIntWrapped64):
int64(value)
elif value is (UIntWrapped32 or LUIntWrapped32):
uint32(value)
elif value is (UIntWrapped64 or LUIntWrapped64):
uint64(value)
elif value is UIntWrapped:
value
else:
{.fatal: "Tried to get the unwrapped value of a non-wrapped type. This should never happen.".}
func encodeBinaryValue(value: VarIntWrapped): auto =
when sizeof(value) == 8:
result = cast[uint64](value)
else:
result = cast[uint32](value)
mixin unwrap
when value is PIntWrapped:
if value.unwrap() < 0:
result = not result
elif value is SIntWrapped:
#This line is the formula exactly as described in the Protobuf docs.
#That said, it's quite verbose.
#The below formula which is actually used is much simpler and possibly faster.
#This is preserved to note it, but not to be used.
#result = (result shl 1) xor cast[type(result)](ashr(value.unwrap(), (sizeof(result) * 8) - 1))
result = result shl 1
if value.unwrap() < 0:
result = not result
elif value is UIntWrapped:
discard
else:
{.fatal: "Tried to get the binary value of an unrecognized VarInt type.".}
func viSizeof(base: VarIntWrapped, raw: uint32 or uint64): int =
when base is PIntWrapped:
if base.unwrap() < 0:
return 10
result = max((log2trunc(raw) + 7) div 7, 1)
func encodeVarInt*(
res: var openarray[byte],
outLen: var int,
value: VarIntWrapped
): VarIntStatus =
#Verify the value fits into the specified encoding.
when value is LUIntWrapped:
when sizeof(value) == 8:
if value.unwrap() shr 63 != 0:
return VarIntStatus.Overflow
#Get the binary value of whatever we're decoding.
#Beyond the above check, LibP2P uses the standard UInt encoding.
#That's why we perform this cast.
var raw = encodeBinaryValue(PInt(value.unwrap()))
else:
var raw = encodeBinaryValue(value)
outLen = viSizeof(value, raw)
#Verify there's enough bytes to store this value.
if res.len < outLen:
return VarIntStatus.Incomplete
#Write the VarInt.
var i = 0
while raw > type(raw)(VAR_INT_VALUE_MASK):
res[i] = byte(raw and type(raw)(VAR_INT_VALUE_MASK)) or VAR_INT_CONTINUATION_MASK
inc(i)
raw = raw shr 7
#If this was a positive number (PInt or UInt), or zig-zagged, we only need to write this last byte.
when value is PIntWrapped:
if value.unwrap() < 0:
#[
To signify this is negative, this should be artifically padded to 10 bytes.
That said, we have to write the final pending byte left in raw, as well as masks until then.
#This iterates up to 9.
We don't immediately write the final pending byte and then loop.
Why? Because if all 9 bytes were used, it'll set the continuation flag when it shouldn't.
If all 9 bytes were used, the last byte is 0 anyways.
By setting raw to 0, which is pointless after the first loop, we avoid two conditionals.
]#
while i < 9:
res[i] = VAR_INT_CONTINUATION_MASK or byte(raw)
inc(i)
raw = 0
else:
res[i] = byte(raw)
else:
res[i] = byte(raw)
func encodeVarInt*(value: VarIntWrapped): seq[byte] =
result = newSeq[byte](10)
var outLen: int
if encodeVarInt(result, outLen, value) != VarIntStatus.Success:
when value is LUIntWrapped:
{.fatal: "LibP2P VarInts require using the following signature: `encodeVarInt(var openarray[byte], outLen: var int, value: VarIntWrapped): VarIntStatus`.".}
else:
doAssert(false)
result.setLen(outLen)
proc encodeVarInt*(stream: OutputStream, value: VarIntWrapped) {.inline.} =
stream.write(encodeVarInt(value))
func decodeBinaryValue[E](
res: var E,
value: uint32 or uint64,
len: int
): VarIntStatus =
when (sizeof(E) != sizeof(value)) and (sizeof(E) != 1):
{.fatal: "Tried to decode a raw binary value into an encoding with a different size. This should never happen.".}
when E is LUIntWrapped:
if res.unwrap() shr ((sizeof(res) * 8) - 1) == 1:
return VarIntStatus.Overflow
res = E(value)
elif E is PIntWrapped:
if len == 10:
type S = type(res.unwrap())
res = E((-S(value)) - 1)
else:
res = E(value)
elif E is SIntWrapped:
type S = type(res.unwrap())
res = E(S(value shr 1) xor -S(value and 0b0000_0001))
elif E is UIntWrapped:
res = E(value)
else:
{.fatal: "Tried to decode a raw binary value into an unrecognized type. This should never happen.".}
return VarIntStatus.Success
func decodeVarInt*(
bytes: openarray[byte],
inLen: var int,
res: var VarIntWrapped
): VarIntStatus =
when sizeof(res) == 8:
type U = uint64
var maxBits = 64
else:
type U = uint32
var maxBits = 32
when (res is LUIntWrapped) and (sizeof(res) == 8):
maxBits = 63
var
value: U
offset = 0'i8
next = VAR_INT_CONTINUATION_MASK
while (next and VAR_INT_CONTINUATION_MASK) != 0:
if inLen == bytes.len:
return VarIntStatus.Incomplete
next = bytes[inLen]
if (next and VAR_INT_VALUE_MASK) == 0:
inLen += 1
offset += 7
continue
if (offset + log2trunc(next and VAR_INT_VALUE_MASK) + 1) > maxBits:
return VarIntStatus.Overflow
value += (next and U(VAR_INT_VALUE_MASK)) shl offset
inLen += 1
offset += 7
return decodeBinaryValue(res, value, inLen)
proc decodeVarInt*[R, E](
stream: InputStream,
returnType: typedesc[R],
encoding: typedesc[E]
): R =
var
bytes: seq[byte]
next: byte = VAR_INT_CONTINUATION_MASK
value: E
inLen: int
while (next and VAR_INT_CONTINUATION_MASK) != 0:
if not stream.readable():
raise newException(ProtobufEOFError, "Stream ended before the VarInt was finished.")
next = stream.read()
bytes.add(next)
if decodeVarInt(bytes, inLen, value) != VarIntStatus.Success:
raise newException(ProtobufMessageError, "Attempted to decode an invalid VarInt.")
doAssert inLen == bytes.len
#Removes a warning.
when value is R:
result = value
else:
result = R(value)

242
protobuf_serialization/reader.nim Normal file
View File

@ -0,0 +1,242 @@
#Parses the Protobuf binary wire protocol into the specified type.
import options
import stew/shims/macros
import faststreams/inputs
import serialization
import internal
import types
proc readVarInt[B, E](
stream: InputStream,
fieldVar: var B,
encoding: E,
key: ProtobufKey
) =
when E is not VarIntWrapped:
{.fatal: "Tried to read a VarInt without a specified encoding. This should never happen.".}
if key.wire != VarInt:
raise newException(ProtobufMessageError, "Invalid wire type for a VarInt.")
#Box the result back up.
box(fieldVar, stream.decodeVarInt(flatType(B), type(E)))
proc readFixed[B](stream: InputStream, fieldVar: var B, key: ProtobufKey) =
type T = flatType(B)
var value: T
when sizeof(T) == 8:
if key.wire != Fixed64:
raise newException(ProtobufMessageError, "Invalid wire type for a Fixed64.")
else:
if key.wire != Fixed32:
raise newException(ProtobufMessageError, "Invalid wire type for a Fixed32.")
stream.decodeFixed(value)
box(fieldVar, value)
include stdlib_readers
proc readValueInternal[T](stream: InputStream, ty: typedesc[T]): T
proc readLengthDelimited[R, B](
stream: InputStream,
rootType: typedesc[R],
fieldName: static string,
fieldVar: var B,
key: ProtobufKey
) =
if key.wire != LengthDelimited:
raise newException(ProtobufMessageError, "Invalid wire type for a length delimited sequence/object.")
var
#We need to specify a bit quantity for decode to be satisfied.
#int64 won't work on int32 systems, as this eventually needs to be casted to int.
#We could just use the proper int size for the system.
#That said, a 2 GB buffer limit isn't a horrible idea from a security perspective.
#If anyone has a valid reason for one, let me know.
#Uses PInt to ensure 31-bits are used, not 32-bits.
len = stream.decodeVarInt(int, PInt(int32))
preResult: flatType(B)
if len < 0:
raise newException(ProtobufMessageError, "Length delimited buffer contained more than 2 GB of data.")
when preResult is CastableLengthDelimitedTypes:
var byteResult: seq[byte] = newSeq[byte](len)
if not stream.readInto(byteResult):
raise newException(ProtobufEOFError, "Couldn't read the length delimited buffer from this stream.")
preResult = cast[type(preResult)](byteResult)
else:
if not stream.readable(len):
raise newException(ProtobufEOFError, "Couldn't read the length delimited buffer from this stream despite expecting one.")
stream.withReadableRange(len, substream):
when preResult is not LengthDelimitedTypes:
{.fatal: "Tried to read a Length Delimited value which we didn't recognize. This should never happen.".}
elif B.isStdlib():
substream.stdlibFromProtobuf(rootType, fieldName, preResult)
elif preResult is (object or tuple):
preResult = substream.readValueInternal(type(preResult))
else:
{.fatal: "Tried to read a Length Delimited type which wasn't actually Length Delimited. This should never happen.".}
box(fieldVar, preResult)
proc setField[T](
value: var T,
stream: InputStream,
key: ProtobufKey
) =
when T is (ref or ptr or Option):
{.fatal: "Ref or Ptr or Option made it to setField. This should never happen.".}
elif T is (seq or set or HashSet):
template merge[I](
stdlib: var (seq[I] or set[I] or HashSet[I]),
value: I
) =
when stdlib is seq:
stdlib.add(value)
else:
stdlib.incl(value)
type U = value.getUnderlyingType()
#Unpacked seq of numbers.
if key.wire != LengthDelimited:
var next: U
when flatType(U) is VarIntWrapped:
stream.readVarInt(next, next, key)
elif flatType(U) is FixedWrapped:
stream.readFixed(next, key)
else:
if true:
raise newException(ProtobufMessageError, "Reading into an unpacked seq yet value is a number.")
merge(value, next)
#Packed seq of numbers/unpacked seq of objects.
else:
when flatType(U) is (VarIntWrapped or FixedWrapped):
var newValues: seq[U]
stream.readLengthDelimited(type(value), "", newValues, key)
for newValue in newValues:
merge(value, newValue)
else:
var
next: flatType(U)
boxed: U
stream.readLengthDelimited(U, "", next, key)
box(boxed, next)
merge(value, boxed)
elif T is not (object or tuple):
when T is VarIntWrapped:
stream.readVarInt(value, value, key)
elif T is FixedWrapped:
stream.readFixed(value, key)
elif T is (PlatformDependentTypes or VarIntTypes or FixedTypes):
{.fatal: "Reading into a number requires specifying both the amount of bits via the type, as well as the encoding format.".}
else:
stream.readLengthDelimited(type(value), "", value, key)
else:
#This iterative approach is extemely poor.
#See https://github.com/kayabaNerve/nim-protobuf-serialization/issues/8.
var
keyNumber = key.number
found = false
enumInstanceSerializedFields(value, fieldName, fieldVar):
discard fieldName
when fieldVar is PlatformDependentTypes:
{.fatal: "Reading into a number requires specifying the amount of bits via the type.".}
if keyNumber == fieldVar.getCustomPragmaVal(fieldNumber):
found = true
var
blank: flatType(fieldVar)
flattened = flatMap(fieldVar).get(blank)
when blank is (seq or set or HashSet):
type U = flattened.getUnderlyingType()
when U is (VarIntWrapped or FixedWrapped):
var castedVar = flattened
elif U is (VarIntTypes or FixedTypes):
when T.hasCustomPragmaFixed(fieldName, pint):
#Nim encounters an error when doing `type C = PInt(U)`.
var
pointless: U
C = PInt(pointless)
elif T.hasCustomPragmaFixed(fieldName, sint):
var
pointless: U
C = SInt(pointless)
elif T.hasCustomPragmaFixed(fieldName, fixed):
var
pointless: U
C = Fixed(pointless)
when flattened is seq:
var castedVar = cast[seq[type(C)]](flattened)
elif flattened is set:
var castedVar = cast[set[type(C)]](flattened)
elif flattened is HashSet:
var castedVar = cast[HashSet[type(C)]](flattened)
else:
var castedVar = flattened
castedVar.setField(stream, key)
flattened = cast[type(flattened)](castedVar)
else:
#Only calculate the encoding for VarInt.
#In every other case, the type is enough.
when flattened is VarIntWrapped:
stream.readVarInt(flattened, flattened, key)
elif flattened is FixedWrapped:
stream.readFixed(flattened, key)
elif flattened is VarIntTypes:
when T.hasCustomPragmaFixed(fieldName, pint):
stream.readVarInt(flattened, PInt(flattened), key)
elif T.hasCustomPragmaFixed(fieldName, sint):
stream.readVarInt(flattened, SInt(flattened), key)
elif T.hasCustomPragmaFixed(fieldName, fixed):
stream.readFixed(flattened, key)
else:
{.fatal: "Encoding pragma specified yet no enoding matched. This should never happen.".}
else:
stream.readLengthDelimited(type(value), fieldName, flattened, key)
box(fieldVar, flattened)
break
if not found:
raise newException(ProtobufMessageError, "Message encoded an unknown field number.")
proc readValueInternal[T](stream: InputStream, ty: typedesc[T]): T =
static: verifySerializable(flatType(T))
while stream.readable():
result.setField(stream, stream.readProtobufKey())
proc readValue*(reader: ProtobufReader, value: var auto) =
try:
if reader.stream.readable():
if reader.keyOverride.isNone():
box(value, reader.stream.readValueInternal(flatType(type(value))))
else:
var preResult: flatType(type(value))
while reader.stream.readable():
preResult.setField(reader.stream, reader.keyOverride.get())
box(value, preResult)
except Exception as e:
raise e
finally:
if reader.closeAfter:
reader.stream.close()

116
protobuf_serialization/stdlib_readers.nim Normal file
View File

@ -0,0 +1,116 @@
#Included by reader.
import sets
import stew/shims/macros
import faststreams
import internal
import types
proc decodeNumber[T, E](
stream: InputStream,
next: var T,
encoding: typedesc[E]
) =
var flattened: flatType(T)
when E is VarIntWrapped:
flattened = stream.decodeVarInt(type(flattened), encoding)
elif E is FixedWrapped:
stream.decodeFixed(flattened)
else:
{.fatal: "Trying to decode a number which isn't wrapped. This should never happen.".}
box(next, flattened)
proc readValue*(reader: ProtobufReader, value: var auto)
proc stdlibFromProtobuf[R](
stream: InputStream,
_: typedesc[R],
unusedFieldName: static string,
value: var string
) =
value = newString(stream.totalUnconsumedBytes)
for c in 0 ..< value.len:
value[c] = char(stream.read())
proc stdlibFromProtobuf[R](
stream: InputStream,
_: typedesc[R],
unusedFieldName: static string,
value: var cstring
) =
var preValue = newString(stream.totalUnconsumedBytes)
for c in 0 ..< preValue.len:
preValue[c] = char(stream.read())
value = preValue
proc stdlibFromProtobuf[R, T](
stream: InputStream,
ty: typedesc[R],
fieldName: static string,
seqInstance: var seq[T]
) =
type fType = flatType(T)
var blank: T
while stream.readable():
seqInstance.add(blank)
when fType is (VarIntWrapped or FixedWrapped):
stream.decodeNumber(seqInstance[^1], type(seqInstance[^1]))
elif fType is VarIntTypes:
when fieldName is "":
{.fatal: "A standard lib type didn't specify the encoding to use for a number.".}
when R.hasCustomPragmaFixed(fieldName, pint):
stream.decodeNumber(seqInstance[^1], PInt(type(seqInstance[^1])))
elif R.hasCustomPragmaFixed(fieldName, sint):
stream.decodeNumber(seqInstance[^1], SInt(type(seqInstance[^1])))
elif R.hasCustomPragmaFixed(fieldName, fixed):
stream.decodeNumber(seqInstance[^1], Fixed(type(seqInstance[^1])))
elif fType is FixedTypes:
when fieldName is "":
{.fatal: "A standard lib type didn't specify the encoding to use for a number.".}
stream.decodeNumber(seqInstance[^1], Fixed(type(seqInstance[^1])))
elif fType is (cstring or string):
var len = stream.decodeVarInt(int, PInt(int32))
if len < 0:
raise newException(ProtobufMessageError, "String longer than 2 GB specified.")
if not stream.readable(len):
raise newException(ProtobufEOFError, "Length delimited buffer is bigger than the rest of the stream.")
stream.withReadableRange(len, substream):
substream.stdlibFromProtobuf(ty, fieldName, seqInstance[^1])
elif fType is CastableLengthDelimitedTypes:
ProtobufReader.init(substream, some(T.wireType), false).readValue(seqInstance[^1])
elif (fType is object) or fType.isStdlib():
let len = stream.decodeVarInt(int, PInt(int32))
if len < 0:
raise newException(ProtobufMessageError, "Length delimited buffer contained more than 2 GB of data.")
elif len == 0:
continue
elif not stream.readable(len):
raise newException(ProtobufEOFError, "Length delimited buffer doesn't have enough data to read the next object.")
stream.withReadableRange(len, substream):
ProtobufReader.init(substream, closeAfter = false).readValue(seqInstance[^1])
else:
{.fatal: "Tried to decode an unrecognized object used in a stdlib type.".}
proc stdlibFromProtobuf[R, T](
stream: InputStream,
ty: typedesc[R],
fieldName: static string,
setInstance: var (set[T] or HashSet[T])
) =
var seqInstance: seq[T]
stream.stdlibFromProtobuf(ty, fieldName, seqInstance)
for value in seqInstance:
setInstance.incl(value)

136
protobuf_serialization/stdlib_writers.nim Normal file
View File

@ -0,0 +1,136 @@
#Included by writer.
import sets
import sequtils
import stew/shims/macros
import internal
import types
proc encodeNumber[T](stream: OutputStream, value: T) =
when value is VarIntWrapped:
stream.encodeVarInt(value)
elif value is FixedWrapped:
stream.encodeFixed(value)
else:
{.fatal: "Trying to encode a number which isn't wrapped. This should never happen.".}
proc stdLibToProtobuf[R](
stream: OutputStream,
_: typedesc[R],
unusedFieldName: static string,
fieldNumber: int,
value: cstring or string
) =
stream.write(cast[seq[byte]]($value))
proc stdlibToProtobuf[R, T](
stream: OutputStream,
ty: typedesc[R],
fieldName: static string,
fieldNumber: int,
arrInstance: openArray[T]
) =
#Get the field number and create a key.
var key: seq[byte]
type fType = flatType(T)
when fType is FixedTypes:
var hasFixed = false
when (R is (object or tuple)) and (not R.isStdlib()):
hasFixed = R.hasCustomPragmaFixed(fieldName, fixed)
when fType is (VarIntTypes or FixedTypes):
when fType is FixedTypes:
if hasFixed:
key = newProtobufKey(
fieldNumber,
when sizeof(fType) == 8:
Fixed64
else:
Fixed32
)
else:
key = newProtobufKey(fieldNumber, VarInt)
else:
key = newProtobufKey(fieldNumber, VarInt)
else:
key = newProtobufKey(fieldNumber, LengthDelimited)
const singleBuffer = type(arrInstance).singleBufferable()
for value in arrInstance:
if not singleBuffer:
stream.write(key)
when fType is (VarIntWrapped or FixedWrapped):
let possibleNumber = flatMap(value)
var blank: fType
stream.encodeNumber(possibleNumber.get(blank))
elif fType is VarIntTypes:
when fieldName is "":
{.fatal: "A standard lib type didn't specify the encoding to use for a number.".}
let possibleNumber = flatMap(value)
var blank: fType
when R.hasCustomPragmaFixed(fieldName, pint):
stream.encodeNumber(PInt(possibleNumber.get(blank)))
elif R.hasCustomPragmaFixed(fieldName, sint):
stream.encodeNumber(SInt(possibleNumber.get(blank)))
elif R.hasCustomPragmaFixed(fieldName, fixed):
stream.encodeNumber(Fixed(possibleNumber.get(blank)))
elif fType is FixedTypes:
when fieldName is "":
{.fatal: "A standard lib type didn't specify the encoding to use for a number.".}
let possibleNumber = flatMap(value)
var blank: fTypeflatType(T)
stream.encodeNumber(Fixed(possibleNumber.get(blank)))
elif fType is (cstring or string):
var cursor = stream.delayVarSizeWrite(5)
let startPos = stream.pos
stream.stdlibToProtobuf(ty, fieldName, fieldNumber, flatMap(value).get(""))
cursor.finalWrite(encodeVarInt(PInt(int32(stream.pos - startPos))))
elif fType is CastableLengthDelimitedTypes:
let toEncode = flatMap(value).get(T(@[]))
if toEncode.len == 0:
return
stream.write(encodeVarInt(PInt(toEncode.len)))
stream.write(cast[seq[byte]](toEncode))
elif (fType is (object or tuple)) or fType.isStdlib():
var cursor = stream.delayVarSizeWrite(5)
let startPos = stream.pos
stream.writeValueInternal(value)
cursor.finalWrite(encodeVarInt(PInt(int32(stream.pos - startPos))))
else:
{.fatal: "Tried to encode an unrecognized object used in a stdlib type.".}
proc stdlibToProtobuf[R, T](
stream: OutputStream,
ty: typedesc[R],
fieldName: static string,
fieldNumber: int,
setInstance: set[T]
) =
var seqInstance: seq[T]
for value in setInstance:
seqInstance.add(value)
stream.stdLibToProtobuf(ty, fieldName, fieldNumber, seqInstance)
proc stdlibToProtobuf[R, T](
stream: OutputStream,
ty: typedesc[R],
fieldName: static string,
fieldNumber: int,
setInstance: HashSet[T]
) {.inline.} =
stream.stdLibToProtobuf(ty, fieldName, fieldNumber, setInstance.toSeq())

50
protobuf_serialization/types.nim Normal file
View File

@ -0,0 +1,50 @@
#Types/common data exported for use outside of this library.
import faststreams
import serialization/errors
import numbers/varint
import numbers/fixed
export varint, fixed
import internal
export fieldNumber, dontOmit
export ProtobufError, ProtobufReadError, ProtobufEOFError, ProtobufMessageError
type
ProtobufFlags* = enum
VarIntLengthPrefix,
UIntLELengthPrefix,
UIntBELengthPrefix
ProtobufWriter* = object
stream*: OutputStream
flags*: set[ProtobufFlags]
ProtobufReader* = ref object
stream*: InputStream
keyOverride*: Option[ProtobufKey]
closeAfter*: bool
func init*(
T: type ProtobufWriter,
stream: OutputStream,
flags: static set[ProtobufFlags] = {}
): T {.inline.} =
T(stream: stream, flags: flags)
func init*(
T: type ProtobufReader,
stream: InputStream,
key: Option[ProtobufKey] = none(ProtobufKey),
closeAfter: bool = true
): T {.inline.} =
T(stream: stream, keyOverride: key, closeAfter: closeAfter)
#This was originally called buffer, and retuned just the output.
#That said, getting the output purges the stream, and doesn't close it.
#Now it's called finish, as there's no reason to keep the stream open at that point.
#A singly function reduces API complexity/expectations on the user.
proc finish*(writer: ProtobufWriter): seq[byte] =
result = writer.stream.getOutput()
writer.stream.close()

219
protobuf_serialization/writer.nim Normal file
View File

@ -0,0 +1,219 @@
#Writes the specified type into a buffer using the Protobuf binary wire format.
import options
import stew/shims/macros
import faststreams/outputs
import serialization
import internal
import types
proc writeVarInt(
stream: OutputStream,
fieldNum: int,
value: VarIntWrapped,
omittable: static bool
) =
let bytes = encodeVarInt(value)
when omittable:
if (bytes.len == 1) and (bytes[0] == 0):
return
stream.writeProtobufKey(fieldNum, VarInt)
stream.write(bytes)
proc writeFixed(
stream: OutputStream,
fieldNum: int,
value: auto,
omittable: static bool
) =
when sizeof(value) == 8:
let wire = Fixed64
else:
let wire = Fixed32
when omittable:
if value.unwrap() == 0:
return
stream.writeProtobufKey(fieldNum, wire)
stream.encodeFixed(value)
proc writeValueInternal[T](stream: OutputStream, value: T)
#stdlib types toProtobuf's. inlined as it needs access to the writeValue function.
include stdlib_writers
proc writeLengthDelimited[T](
stream: OutputStream,
fieldNum: int,
rootType: typedesc[T],
fieldName: static string,
flatValue: LengthDelimitedTypes,
omittable: static bool
) =
const stdlib = type(flatValue).isStdlib()
var cursor = stream.delayVarSizeWrite(10)
let startPos = stream.pos
#Byte seqs.
when flatValue is CastableLengthDelimitedTypes:
if flatValue.len == 0:
cursor.finalWrite([])
return
stream.write(cast[seq[byte]](flatValue))
#Standard lib types which use custom converters, instead of encoding the literal Nim representation.
elif stdlib:
stream.stdlibToProtobuf(rootType, fieldName, fieldNum, flatValue)
#Nested object which even if the sub-value is empty, should be encoded as long as it exists.
elif rootType.isPotentiallyNull():
writeValueInternal(stream, flatValue)
#Object which should only be encoded if it has data.
elif flatValue is (object or tuple):
writeValueInternal(stream, flatValue)
else:
{.fatal: "Tried to write a Length Delimited type which wasn't actually Length Delimited.".}
const singleBuffer = type(flatValue).singleBufferable()
if (
(
#The underlying type of the standard library container is packable.
singleBuffer or (
#This is a object, not a seq or something converted to a seq (stdlib type).
(not stdlib) and (flatValue is (object or tuple))
)
) and (
#The length changed, meaning this object is empty.
(stream.pos != startPos) or
#The object is empty, yet it exists, which is important as it can not exist.
rootType.isPotentiallyNull()
)
):
cursor.finalWrite(newProtobufKey(fieldNum, LengthDelimited) & encodeVarInt(PInt(int32(stream.pos - startPos))))
else:
when omittable:
cursor.finalWrite([])
else:
cursor.finalWrite(newProtobufKey(fieldNum, LengthDelimited) & encodeVarInt(PInt(int32(0))))
proc writeFieldInternal[T, R](
stream: OutputStream,
fieldNum: int,
value: T,
rootType: typedesc[R],
fieldName: static string
) =
static: verifySerializable(flatType(T))
let flattenedOption = value.flatMap()
if flattenedOption.isNone():
return
let flattened = flattenedOption.get()
when (flatType(R) is not object) or (fieldName == ""):
const omittable = true
else:
const omittable = not flatType(R).hasCustomPragmaFixed(fieldName, dontOmit)
when flattened is VarIntWrapped:
stream.writeVarInt(fieldNum, flattened, omittable)
elif flattened is FixedWrapped:
stream.writeFixed(fieldNum, flattened, omittable)
else:
stream.writeLengthDelimited(fieldNum, R, fieldName, flattened, omittable)
proc writeField*[T](
writer: ProtobufWriter,
fieldNum: int,
value: T
) {.inline.} =
writer.stream.writeFieldInternal(fieldNum, value, type(value), "")
proc writeValueInternal[T](stream: OutputStream, value: T) =
static: verifySerializable(flatType(T))
let flattenedOption = value.flatMap()
if flattenedOption.isNone():
return
let flattened = flattenedOption.get()
when type(flattened).isStdlib():
stream.writeFieldInternal(1, flattened, type(value), "")
elif flattened is (object or tuple):
enumInstanceSerializedFields(flattened, fieldName, fieldVal):
discard fieldName
const fieldNum = getCustomPragmaVal(fieldVal, fieldNumber)
let flattenedFieldOption = fieldVal.flatMap()
if flattenedFieldOption.isSome():
let flattenedField = flattenedFieldOption.get()
when flattenedField is ((not (VarIntWrapped or FixedWrapped)) and (VarIntTypes or FixedTypes)):
when flattenedField is VarIntTypes:
const
hasPInt = flatType(value).hasCustomPragmaFixed(fieldName, pint)
hasSInt = flatType(value).hasCustomPragmaFixed(fieldName, sint)
hasFixed = flatType(value).hasCustomPragmaFixed(fieldName, fixed)
when hasPInt:
stream.writeFieldInternal(fieldNum, PInt(flattenedField), type(value), fieldName)
elif hasSInt:
stream.writeFieldInternal(fieldNum, SInt(flattenedField), type(value), fieldName)
elif hasFixed:
stream.writeFieldInternal(fieldNum, Fixed(flattenedField), type(value), fieldName)
else:
{.fatal: "Encoding pragma specified yet no enoding matched. This should never happen.".}
elif flattenedField is FixedTypes:
stream.writeFieldInternal(fieldNum, flattenedField, type(value), fieldName)
else:
{.fatal: "Attempting to handle an unknown number type. This should never happen.".}
else:
stream.writeFieldInternal(fieldNum, flattenedField, type(value), fieldName)
else:
stream.writeFieldInternal(1, flattened, type(value), "")
proc writeValue*[T](writer: ProtobufWriter, value: T) =
var
cursor: VarSizeWriteCursor
startPos: int
if (
writer.flags.contains(VarIntLengthPrefix) or
writer.flags.contains(UIntLELengthPrefix) or
writer.flags.contains(UIntBELengthPrefix)
):
cursor = writer.stream.delayVarSizeWrite(5)
startPos = writer.stream.pos
writer.stream.writeValueInternal(value)
if (
writer.flags.contains(VarIntLengthPrefix) or
writer.flags.contains(UIntLELengthPrefix) or
writer.flags.contains(UIntBELengthPrefix)
):
var len = uint32(writer.stream.pos - startPos)
if len == 0:
cursor.finalWrite([])
elif writer.flags.contains(VarIntLengthPrefix):
var viLen = encodeVarInt(PInt(len))
if viLen.len == 0:
cursor.finalWrite([byte(0)])
else:
cursor.finalWrite(viLen)
elif writer.flags.contains(UIntLELengthPrefix):
var temp: array[sizeof(len), byte]
for i in 0 ..< sizeof(len):
temp[i] = byte(len and LAST_BYTE)
len = len shr 8
cursor.finalWrite(temp)
elif writer.flags.contains(UIntBELengthPrefix):
var temp: array[sizeof(len), byte]
for i in 0 ..< sizeof(len):
temp[i] = byte(len shr ((sizeof(len) - 1) * 8))
len = len shl 8
cursor.finalWrite(temp)

2
tests/config.nims
View File

@ -1 +1 @@
switch("path", "$projectDir/../")
switch("threads", "on")

5
tests/fail/test_cstring.nim Normal file
View File

@ -0,0 +1,5 @@
import tables
import ../../protobuf_serialization
discard Protobuf.encode(cstring("Testing string."))

6
tests/fail/test_invalid_byte_encoding.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type InvalidByteEncoding = object
x {.sint, fieldNumber: 1.}: uint8
discard Protobuf.encode(InvalidByteEncoding())

6
tests/fail/test_invalid_float_bits.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type InvalidFloatBits = object
x {.pfloat32, fieldNumber: 1.}: float64
discard Protobuf.encode(InvalidFloatBits())

6
tests/fail/test_invalid_float_encoding.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type InvalidFloatEncoding = object
x {.pint, pfloat32, fieldNumber: 1.}: float32
discard Protobuf.encode(InvalidFloatEncoding())

10
tests/fail/test_invalid_nested_object.nim Normal file
View File

@ -0,0 +1,10 @@
import ../../protobuf_serialization
type
X = object
y {.pint, sint, fieldNumber: 1.}: int32
A = object
b {.fieldNumber: 1.}: X
discard Protobuf.encode(A())

6
tests/fail/test_invalid_uint_encoding.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type InvalidUIntEncoding = object
x {.sint, fieldNumber: 1.}: uint32
discard Protobuf.encode(InvalidUIntEncoding())

6
tests/fail/test_multiple_int_encodings.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type MultipleIntEncodings = object
x {.pint, sint, fieldNumber: 1.}: int32
discard Protobuf.encode(MultipleIntEncodings())

6
tests/fail/test_multiple_uint_encodings.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type MultipleUIntEncodings = object
x {.pint, fixed, fieldNumber: 1.}: uint32
discard Protobuf.encode(MultipleUIntEncodings())

6
tests/fail/test_negative_field_number.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type NegativeFieldNumber = object
x {.fieldNumber: -1.}: bool
discard Protobuf.encode(NegativeFieldNumber())

6
tests/fail/test_no_field_number.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type NoFieldNumber = object
x: bool
discard Protobuf.encode(NoFieldNumber())

6
tests/fail/test_no_int_encoding.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type NoIntEncoding = object
x {.fieldNumber: 1.}: int32
discard Protobuf.encode(NoIntEncoding())

6
tests/fail/test_no_uint_encoding.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type NoUIntEncoding = object
x {.fieldNumber: 1.}: uint32
discard Protobuf.encode(NoUIntEncoding())

7
tests/fail/test_reused_field_number.nim Normal file
View File

@ -0,0 +1,7 @@
import ../../protobuf_serialization
type ReusedFieldNumber = object
x {.fieldNumber: 1.}: bool
y {.fieldNumber: 1.}: bool
discard Protobuf.encode(ReusedFieldNumber())

6
tests/fail/test_specified_byte_encoding.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type SpecifiedByteEncoding = object
x {.pint, fieldNumber: 1.}: uint8
discard Protobuf.encode(SpecifiedByteEncoding())

6
tests/fail/test_specified_float_encoding.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type SpecifiedFloatEncoding = object
x {.fixed, fieldNumber: 1.}: float32
discard Protobuf.encode(SpecifiedFloatEncoding())

5
tests/fail/test_table.nim Normal file
View File

@ -0,0 +1,5 @@
import tables
import ../../protobuf_serialization
discard Protobuf.encode([(5, 5)].toTable())

10
tests/fail/test_too_high_field_number.nim Normal file
View File

@ -0,0 +1,10 @@
import ../../protobuf_serialization
#The field number must fix into 2^28.
#A signed int has the first bit unavailable, lowering the available bits to 2^31.
#Then encoding the wire type requires shifting left 3 bits.
#That leaves you with 2^27 for consistency on all platforms without slow extensions
type TooHighFieldNumber = object
x {.fieldNumber: 268435457.}: bool
discard Protobuf.encode(TooHighFieldNumber())

3
tests/fail/test_tuple.nim Normal file
View File

@ -0,0 +1,3 @@
import ../../protobuf_serialization
discard Protobuf.encode((5, 5))

6
tests/fail/test_unspecified_float_bits.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type UnspecifiedFloatBits = object
x {.fieldNumber: 1.}: float64
discard Protobuf.encode(UnspecifiedFloatBits())

6
tests/fail/test_unspecified_int_bits.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type UnspecifiedIntBits = object
x {.sint, fieldNumber: 1.}: int
discard Protobuf.encode(UnspecifiedIntBits())

6
tests/fail/test_unspecified_uint_bits.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type UnspecifiedUIntBits = object
x {.pint, fieldNumber: 1.}: uint
discard Protobuf.encode(UnspecifiedUIntBits())

6
tests/fail/test_zero_field_number.nim Normal file
View File

@ -0,0 +1,6 @@
import ../../protobuf_serialization
type ZeroFieldNumber = object
x {.fieldNumber: 0.}: bool
discard Protobuf.encode(ZeroFieldNumber())

20
tests/test_all.nim Normal file
View File

@ -0,0 +1,20 @@
{.warning[UnusedImport]: off}
import ../protobuf_serialization
import
test_bool,
test_lint,
test_fixed,
test_length_delimited,
test_objects,
test_empty,
test_options,
test_stdlib,
test_different_types,
test_thirty_three_fields
#Test internal types aren't exported.
#There's just not a good place for this to go.
when defined(PIntWrapped32):
assert(false, "Internal types are being exported.")

56
tests/test_bool.nim Normal file
View File

@ -0,0 +1,56 @@
import unittest
import ../protobuf_serialization
type
PIntType = object
x {.pint, fieldNumber: 1.}: int32
UIntType = object
x {.pint, fieldNumber: 1.}: uint32
SIntType = object
x {.sint, fieldNumber: 1.}: int32
BoolType = object
x {.fieldNumber: 1.}: bool
proc writeRead[W, R](toWrite: W, value: R) =
check Protobuf.decode(Protobuf.encode(toWrite), R) == value
suite "Test Boolean Encoding/Decoding":
test "Can encode/decode boolean without subtype specification":
writeRead(true, true)
writeRead(false, false)
writeRead(BoolType(x: true), BoolType(x: true))
writeRead(BoolType(x: false), BoolType(x: false))
#Skipping subtype specification only works when every encoding has the same truthiness.
#That's what this tests. It should be noted 1 encodes as 1/1/2 for the following.
test "Can encode/decode boolean as signed VarInt":
writeRead(PInt(0'i32), false)
writeRead(PInt(0'i64), false)
writeRead(PInt(1'i32), true)
writeRead(PInt(1'i64), true)
writeRead(PIntType(x: 1), BoolType(x: true))
writeRead(PIntType(x: 0), BoolType(x: false))
test "Can encode/decode boolean as unsigned VarInt":
writeRead(PInt(0'u32), false)
writeRead(PInt(0'u64), false)
writeRead(PInt(1'u32), true)
writeRead(PInt(1'u64), true)
writeRead(UIntType(x: 1), BoolType(x: true))
writeRead(UIntType(x: 0), BoolType(x: false))
test "Can encode/decode boolean as zig-zagged VarInt":
writeRead(SInt(0'i32), false)
writeRead(SInt(0'i64), false)
writeRead(SInt(1'i32), true)
writeRead(SInt(1'i64), true)
writeRead(SIntType(x: 1), BoolType(x: true))
writeRead(SIntType(x: 0), BoolType(x: false))

31
tests/test_different_types.nim Normal file
View File

@ -0,0 +1,31 @@
import unittest
import ../protobuf_serialization
proc writeRead[W, R](toWrite: W, readAs: typedesc[R]) =
expect ProtobufMessageError:
discard Protobuf.decode(Protobuf.encode(toWrite), R)
suite "Test Encoding X and decoding into Y":
test "* into VarInt":
#Test the Fixed32 and Fixed64 wire types.
writeRead(Fixed(5'u32), SInt(int32))
writeRead(Fixed(5'u64), SInt(int32))
#LengthDelimited.
writeRead("Test string.", SInt(int32))
test "* into Fixed":
#VarInt.
writeRead(SInt(5'i32), Fixed(uint32))
#LengthDelimited.
writeRead("Test string.", Fixed(uint32))
test "* into LengthDelimited":
#VarInt.
writeRead(SInt(5'i32), string)
#Fixed.
writeRead(Fixed(5'u32), string)
writeRead(Fixed(5'u64), string)

59
tests/test_empty.nim Normal file
View File

@ -0,0 +1,59 @@
import unittest
import ../protobuf_serialization
from test_objects import DistinctInt, `==`
type DistinctTypeSerialized = SInt(int32)
DistinctInt.borrowSerialization(DistinctTypeSerialized)
type
X = object
Y = object
a {.pint, fieldNumber: 1.}: int32
Z = object
b {.dontSerialize.}: string
DOY = object
a {.pint, dontOmit, fieldNumber: 1.}: int32
proc writeEmpty[T](value: T) =
check Protobuf.encode(value).len == 0
suite "Test Encoding of Empty Objects/Values":
test "Empty boolean":
writeEmpty(false)
test "Empty signed VarInt":
writeEmpty(PInt(0'i32))
writeEmpty(PInt(0'i64))
test "Empty unsigned VarInt":
writeEmpty(PInt(0'u32))
writeEmpty(PInt(0'u64))
test "Empty zigzagged VarInt":
writeEmpty(SInt(0'i32))
writeEmpty(SInt(0'i64))
test "Empty Fixed64":
writeEmpty(Fixed(0'i64))
writeEmpty(Fixed(0'u64))
writeEmpty(Float64(0'f64))
test "Empty length-delimited":
writeEmpty("")
test "Empty object":
writeEmpty(X())
writeEmpty(Y())
writeEmpty(Z(b: "abc"))
check Protobuf.encode(DOY()).len == 2
test "Empty distinct type":
writeEmpty(DistinctInt(0))
test "Empty Fixed32":
writeEmpty(Fixed(0'i32))
writeEmpty(Fixed(0'u32))
writeEmpty(Float32(0'f32))

27
tests/test_fixed.nim Normal file
View File

@ -0,0 +1,27 @@
import unittest
import ../protobuf_serialization
from ../protobuf_serialization/internal import unwrap
proc writeRead(x: auto) =
when sizeof(x) == 4:
check cast[uint32](Protobuf.decode(Protobuf.encode(x), type(x))) == cast[uint32](x)
else:
check cast[uint64](Protobuf.decode(Protobuf.encode(x), type(x))) == cast[uint64](x)
suite "Test Fixed Encoding/Decoding":
test "Can encode/decode int":
writeRead(Fixed(2'i32))
writeRead(Fixed(3'i64))
writeRead(Fixed(-4'i32))
writeRead(Fixed(-5'i64))
test "Can encode/decode uint":
writeRead(Fixed(6'u32))
writeRead(Fixed(7'u64))
test "Can encode/decode float":
writeRead(Float32(8.90123'f32))
writeRead(Float64(4.56789'f64))
writeRead(Float32(-0.1234'f32))
writeRead(Float64(-5.6789'f64))

69
tests/test_length_delimited.nim Normal file
View File

@ -0,0 +1,69 @@
import math
import unittest
import ../protobuf_serialization
#func cstrlen(x: cstring): csize_t {.header: "string.h", importc: "strlen".}
suite "Test Length Delimited Encoding/Decoding":
test "Can encode/decode string":
let
str = "Testing string.\0"
output = Protobuf.encode(str)
check output == @[byte(10), byte(str.len), 84, 101, 115, 116, 105, 110, 103, 32, 115, 116, 114, 105, 110, 103, 46, 0]
check Protobuf.decode(output, type(string)) == str
test "Can encode/decode char seq":
let
charSeq = cast[seq[char]]("Testing string.\0")
output = Protobuf.encode(charSeq)
check output == @[byte(10), byte(charSeq.len), 84, 101, 115, 116, 105, 110, 103, 32, 115, 116, 114, 105, 110, 103, 46, 0]
check Protobuf.decode(output, type(seq[char])) == charSeq
test "Can encode/decode byte seq":
let
byteSeq = cast[seq[byte]]("Testing string.\0")
output = Protobuf.encode(byteSeq)
check output == @[byte(10), byte(byteSeq.len), 84, 101, 115, 116, 105, 110, 103, 32, 115, 116, 114, 105, 110, 103, 46, 0]
check Protobuf.decode(output, type(seq[byte])) == byteSeq
test "Can encode/decode byte seq seq":
let
byteSeqSeq = cast[seq[seq[byte]]](@[
"Testing string.\0",
"Other value!",
"Shares nothing@",
])
output = Protobuf.encode(byteSeqSeq)
check output == @[
byte(10), byte(byteSeqSeq[0].len),
84, 101, 115, 116, 105, 110, 103, 32, 115, 116, 114, 105, 110, 103, 46, 0,
10, byte(byteSeqSeq[1].len),
79, 116, 104, 101, 114, 32, 118, 97, 108, 117, 101, 33,
10, byte(byteSeqSeq[2].len),
83, 104, 97, 114, 101, 115, 32, 110, 111, 116, 104, 105, 110, 103, 64
]
check Protobuf.decode(output, type(seq[seq[byte]])) == byteSeqSeq
test "Can encode/decode bool seq":
let
boolSeq = @[true, false, false, true, true, true, true, false, true, false, false, false]
output = Protobuf.encode(boolSeq)
check output == @[byte(10), byte(boolSeq.len), 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0]
check Protobuf.decode(output, type(seq[bool])) == boolSeq
#[test "Decoding a string/cstring doesn't remove the null terminator":
let str = "Testing string."
check cstrlen(Protobuf.decode(Protobuf.encode(str), string)) == csize_t(str.len)
check cstrlen(Protobuf.decode(Protobuf.encode(str), cstring)) == csize_t(str.len)
check cstrlen(Protobuf.decode(Protobuf.encode(cstring(str)), string)) == csize_t(str.len)
check cstrlen(Protobuf.decode(Protobuf.encode(cstring(str)), cstring)) == csize_t(str.len)]#
test "Can encode a string which has a length which requires three bytes to encode":
let
x = newString(2 ^ 15)
vi = Protobuf.encode(PInt(x.len))
encoded = Protobuf.encode(x)
check encoded[1 ..< vi.len] == vi[1 ..< vi.len]
check Protobuf.decode(encoded, string) == x

52
tests/test_lint.nim Normal file
View File

@ -0,0 +1,52 @@
#[import unittest
import ../protobuf_serialization
from ../protobuf_serialization/internal import unwrap
proc writeRead(x: auto) =
let encoded = Protobuf.encode(LInt(x))
#LInt sets a cap of 10 bytes. That said, the wire byte is prefixed.
#Hence the 11.
check encoded.len < 11
check Protobuf.decode(encoded, type(LInt(x))).unwrap() == x
suite "Test LInt Encoding/Decoding":
test "Can encode/decode uint":
writeRead(0'u32)
writeRead(1'u32)
writeRead(254'u32)
writeRead(255'u32)
writeRead(256'u32)
writeRead(1'u64 shl 62)
test "Can detect too large uints":
expect ProtobufWriteError:
writeRead(1'u64 shl 63)
#Following tests also work for VarInts in general.
#We don't have a dedicated VarInt suite.
test "Can detect overflown byte buffers":
var
bytes = @[byte(255), 255, 255, 255, 127]
inLen: int
res32: LInt(uint32)
res64: LInt(uint32)
check decodeVarInt(bytes, inLen, res32) == VarIntStatus.Overflow
bytes = @[byte(255), 255, 255, 255, 255, 255, 255, 255, 255, 127]
check decodeVarInt(bytes, inLen, res64) == VarIntStatus.Overflow
test "Can handle the highest/lowest value for each encoding":
template testHighLow(Encoding: untyped, ty: typed) =
check Protobuf.decode(Protobuf.encode(Encoding(high(ty))), Encoding(ty)).unwrap() == high(ty)
check Protobuf.decode(Protobuf.encode(Encoding(low(ty))), Encoding(ty)).unwrap() == low(ty)
testHighLow(PInt, int32)
testHighLow(PInt, int64)
testHighLow(PInt, uint32)
testHighLow(PInt, uint64)
testHighLow(SInt, int32)
testHighLow(SInt, int64)
check Protobuf.decode(Protobuf.encode(LInt(high(uint32))), LInt(uint32)).unwrap() == high(uint32)
check Protobuf.decode(Protobuf.encode(LInt(high(uint64) shr 1)), LInt(uint64)).unwrap() == (high(uint64) shr 1)
]#

230
tests/test_objects.nim Normal file
View File

@ -0,0 +1,230 @@
import unittest
import ../protobuf_serialization
type
TestEnum = enum
NegTwo = -2, NegOne, Zero, One, Two
DistinctInt* = distinct int32
Basic = object
a {.pint, fieldNumber: 1.}: uint64
b {.fieldNumber: 2.}: string
c {.fieldNumber: 3.}: char
Wrapped = object
d {.sint, fieldNumber: 1.}: int32
e {.sint, fieldNumber: 2.}: int64
f {.fieldNumber: 3.}: Basic
g {.fieldNumber: 4.}: string
h {.fieldNumber: 5.}: bool
Nested* = ref object
child* {.fieldNumber: 1.}: Nested
data* {.fieldNumber: 2.}: string
Circular = ref object
child {.fieldNumber: 1.}: Circular
Pointered = object
x {.sint, fieldNumber: 1.}: ptr int32
discard Protobuf.supports(Basic)
discard Protobuf.supports(Wrapped)
discard Protobuf.supports(Nested)
discard Protobuf.supports(Circular)
type DistinctTypeSerialized = SInt(int32)
DistinctInt.borrowSerialization(DistinctTypeSerialized)
proc `==`*(lhs: DistinctInt, rhs: DistinctInt): bool {.borrow.}
proc `==`*(lhs: Nested, rhs: Nested): bool =
var
lastLeft = lhs
lastRight = rhs
while not lastLeft.isNil:
if lastRight.isNil:
return false
if lastLeft.data != lastRight.data:
return false
lastLeft = lastLeft.child
lastRight = lastRight.child
if not lastRight.isNil:
return false
result = true
suite "Test Object Encoding/Decoding":
#The following three tests don't actually test formal objects.
#They test user-defined types. This is just the best place for these tests.
test "Can encode/decode enums":
template enumTest(value: TestEnum, integer: int): untyped =
let output = Protobuf.encode(SInt(value))
if integer == 0:
check output.len == 0
else:
check output == @[byte(8), byte(integer)]
check TestEnum(Protobuf.decode(output, type(SInt(TestEnum)))) == value
enumTest(NegTwo, 3)
enumTest(NegOne, 1)
enumTest(Zero, 0)
enumTest(One, 2)
enumTest(Two, 4)
test "Can encode/decode distinct types":
let x: DistinctInt = 5.DistinctInt
check Protobuf.decode(Protobuf.encode(x), type(DistinctInt)) == x
#[test "Can encode/decode tuples":
let
unnamed: (
SInt(int32),
PInt(uint32),
bool,
string,
bool
) = (SInt(5'i32), PInt(3'u32), true, "abc", false)
unnamedRead = Protobuf.decode(Protobuf.encode(unnamed), type(unnamed))
named: tuple[
a: SInt(int32),
b: PInt(uint32),
c: bool,
d: string,
e: bool
] = (
a: SInt(6'i32),
b: PInt(4'u32),
c: false,
d: "def",
e: true
)
namedRead = Protobuf.decode(Protobuf.encode(named), type(named))
check int32(unnamedRead[0]) == int32(unnamed[0])
check uint32(unnamedRead[1]) == uint32(unnamed[1])
check unnamedRead[2] == unnamed[2]
check unnamedRead[3] == unnamed[3]
check unnamedRead[4] == unnamed[4]
check int32(namedRead.a) == int32(named.a)
check uint32(namedRead.b) == uint32(named.b)
check namedRead.c == named.c
check namedRead.d == named.d
check namedRead.e == named.e]#
test "Can encode/decode objects":
let
obj = Basic(a: 100, b: "Test string.", c: 'C')
encoded = Protobuf.encode(obj)
check Protobuf.decode(encoded, Basic) == obj
#Test VarInt length prefixing as well.
let prefixed = Protobuf.encode(obj, {VarIntLengthPrefix})
var
inLen: int
res: PInt(int32)
check prefixed.len > encoded.len
check decodeVarInt(prefixed[0 ..< (prefixed.len - encoded.len)], inLen, res) == VarIntStatus.Success
check inLen == (prefixed.len - encoded.len)
check res.unwrap() == encoded.len
test "Can encode/decode a wrapper object":
let obj = Wrapped(
d: 300,
e: 200,
f: Basic(a: 100, b: "Test string.", c: 'C'),
g: "Other test string.",
h: true
)
check Protobuf.decode(Protobuf.encode(obj), type(Wrapped)) == obj
test "Can encode/decode partial object":
let
obj = Wrapped(
d: 300,
e: 200,
f: Basic(a: 100, b: "Test string.", c: 'C'),
g: "Other test string.",
h: true
)
writer = ProtobufWriter.init(memoryOutput())
writer.writeField(1, SInt(obj.d))
writer.writeField(3, obj.f)
writer.writeField(4, obj.g)
let result = Protobuf.decode(writer.finish(), type(Wrapped))
check result.d == obj.d
check result.f == obj.f
check result.g == obj.g
check result.e == 0
check result.h == false
test "Can encode/decode out of order object":
let
obj = Wrapped(
d: 300,
e: 200,
f: Basic(a: 100, b: "Test string.", c: 'C'),
g: "Other test string.",
h: true
)
writer = ProtobufWriter.init(memoryOutput())
writer.writeField(3, obj.f)
writer.writeField(1, SInt(obj.d))
writer.writeField(2, SInt(obj.e))
writer.writeField(5, obj.h)
writer.writeField(4, obj.g)
check Protobuf.decode(writer.finish(), type(Wrapped)) == obj
test "Can read repeated fields":
let
writer = ProtobufWriter.init(memoryOutput())
basic: Basic = Basic(b: "Initial string.")
repeated = "Repeated string."
writer.writeField(2, basic.b)
writer.writeField(2, repeated)
check Protobuf.decode(writer.finish(), type(Basic)) == Basic(b: repeated)
test "Can read nested objects":
let obj: Nested = Nested(
child: Nested(
data: "Child data."
),
data: "Parent data."
)
check Protobuf.decode(Protobuf.encode(obj), type(Nested)) == obj
test "Can read pointered objects":
var ptrd = Pointered()
ptrd.x = cast[ptr int32](alloc0(sizeof(int32)))
ptrd.x[] = 5
check Protobuf.decode(Protobuf.encode(ptrd), Pointered).x[] == ptrd.x[]
var ptrPtrd = addr ptrd
ptrPtrd.x = cast[ptr int32](alloc0(sizeof(int32)))
ptrPtrd.x[] = 8
check Protobuf.decode(Protobuf.encode(ptrPtrd), ptr Pointered).x[] == ptrPtrd.x[]
#[
This test has been commented for being pointless.
The reason this fails is because it detects a field number of 0, which is invalid.
Any valid field will be considered valid, as long as the length is correct.
If the length isn't, it's incorrect.
That said, those are two different things than remaining data.
test "Doesn't allow remaining data in the buffer":
expect ProtobufReadError:
discard Protobuf.decode(Protobuf.encode(SInt(5)) & @[byte(1)], type(SInt(int32)))
expect ProtobufReadError:
discard Protobuf.decode(Protobuf.encode(Basic(a: 100, b: "Test string.", c: 'C')) & @[byte(1)], type(Basic))
]#
test "Doesn't allow unknown fields":
expect ProtobufMessageError:
discard Protobuf.decode((Protobuf.encode(Basic(a: 100, b: "Test string.", c: 'C')) & @[byte(4 shl 3)]), type(Basic))

139
tests/test_options.nim Normal file
View File

@ -0,0 +1,139 @@
import options
import unittest
import ../protobuf_serialization
from ../protobuf_serialization/internal import VarIntWrapped, FixedWrapped, unwrap, flatType, flatMap
from test_objects import DistinctInt, `==`
type
Basic = object
x {.sint, fieldNumber: 1.}: int32
Wrapped = object
y {.sint, fieldNumber: 1.}: Option[int32]
Nested = ref object
child {.fieldNumber: 1.}: Option[Nested]
z {.fieldNumber: 2.}: Option[Wrapped]
proc `==`*(lhs: Nested, rhs: Nested): bool =
lhs.z == rhs.z
template testNone[T](ty: typedesc[T]) =
let output = Protobuf.encode(none(ty))
check output.len == 0
check Protobuf.decode(output, type(Option[T])).isNone()
template testSome[T](value: T) =
let output = Protobuf.encode(some(value))
check output == Protobuf.encode(flatMap(value))
when flatType(T) is (VarIntWrapped or FixedWrapped):
check Protobuf.decode(output, type(Option[T])).get().unwrap() == some(value).get().unwrap()
else:
check Protobuf.decode(output, type(Option[T])) == some(value)
suite "Test Encoding/Decoding of Options":
test "Option boolean":
testNone(bool)
testSome(true)
test "Option signed VarInt":
testNone(PInt(int32))
testSome(PInt(5'i32))
testSome(PInt(-5'i32))
test "Option unsigned VarInt":
testNone(PInt(uint32))
testSome(PInt(5'u32))
test "Option zigzagged VarInt":
testNone(SInt(int32))
testSome(SInt(5'i32))
testSome(SInt(-5'i32))
test "Option Fixed":
template fixedTest[T](value: T): untyped =
testNone(type(T))
testSome(value)
fixedTest(Fixed(5'i64))
fixedTest(Fixed(-5'i64))
fixedTest(Fixed(5'i32))
fixedTest(Fixed(-5'i32))
fixedTest(Fixed(5'u64))
fixedTest(Fixed(5'u32))
fixedTest(Float64(5.5'f64))
fixedTest(Float64(-5.5'f64))
fixedTest(Float32(5.5'f32))
fixedTest(Float32(-5.5'f32))
test "Option length-delimited":
testNone(string)
testNone(seq[byte])
testSome("Testing string.")
testSome(@[byte(0), 1, 2, 3, 4])
test "Option object":
testNone(Basic)
testNone(Wrapped)
testSome(Basic(x: 5'i32))
testSome(Wrapped(y: some(5'i32)))
test "Option ref":
#This is in a block, manually expanded, with a pointless initial value.
#Why?
#https://github.com/nim-lang/Nim/issues/14387
block one4387:
var option = some(Nested())
option = none(Nested)
let output = Protobuf.encode(option)
check output.len == 0
check Protobuf.decode(output, type(Option[Nested])).isNone()
testSome(Nested(
child: some(Nested(
child: none(Nested),
z: none(Wrapped)
)),
z: none(Wrapped)
))
testSome(Nested(
child: none(Nested),
z: some(Wrapped(y: some(5'i32)))
))
testSome(Nested(
child: some(Nested(
child: none(Nested),
z: some(Wrapped(y: some(5'i32)))
)),
z: some(Wrapped(y: some(5'i32)))
))
testSome(Nested(
child: some(Nested(
z: some(Wrapped(y: some(5'i32)))
)),
z: some(Wrapped(y: some(5'i32)))
))
test "Option ptr":
testNone(ptr Basic)
let basicInst = Basic(x: 5'i32)
let output = Protobuf.encode(some(basicInst))
check output == Protobuf.encode(flatMap(basicInst))
check Protobuf.decode(output, Option[ptr Basic]).get()[] == basicInst
#This was banned at one point in this library's lifetime.
#It should work now.
test "Option Option":
testNone(string)
testSome(some("abc"))

280
tests/test_serialization.nim
View File

@ -1,280 +0,0 @@
import unittest
import sequtils
import protobuf_serialization
type
MyEnum = enum
ME1, ME2, ME3
Test1 = object
a: uint
b: string
c: char
Test3 = object
g {.sfixed32.}: int
h: int
i: Test1
j: string
k: bool
l: MyInt
MyInt = distinct int
proc to*(bytes: var seq[byte], ty: typedesc[MyInt]): MyInt =
var value: int
var shiftAmount = 0
for i in 0 ..< len(bytes):
value += int(bytes[i]) shl shiftAmount
shiftAmount += 8
result = MyInt(value)
proc toBytes*(value: MyInt): seq[byte] =
var value = value.int
while value > 0:
result.add byte(value and 0b1111_1111)
value = value shr 8
proc `==`(a, b: MyInt): bool {.borrow.}
suite "Test Varint Encoding":
test "Can encode/decode enum field":
var proto = newProtoBuffer()
var bytesProcessed: int
proto.encodeField(ME3)
proto.encodeField(ME2)
var output = proto.output
assert output == @[8.byte, 4, 16, 2]
var offset = 0
let decodedME3 = decodeField(output, MyEnum, offset, bytesProcessed)
assert decodedME3.value == ME3
assert decodedME3.index == 1
let decodedME2 = decodeField(output, MyEnum, offset, bytesProcessed)
assert decodedME2.value == ME2
assert decodedME2.index == 2
test "Can encode/decode negative number field":
var proto = newProtoBuffer()
let num = -153452
var bytesProcessed: int
proto.encodeField(num)
var output = proto.output
assert output == @[8.byte, 215, 221, 18]
var offset = 0
let decoded = decodeField(output, int, offset, bytesProcessed)
assert decoded.value == num
assert decoded.index == 1
test "Can encode/decode distinct number field":
var proto = newProtoBuffer()
let num = 114151.MyInt
var bytesProcessed: int
proto.encodeField(num)
var output = proto.output
assert output == @[10.byte, 3, 231, 189, 1]
var offset = 0
let decoded = decodeField(output, MyInt, offset, bytesProcessed)
assert decoded.value.int == num.int
assert decoded.index == 1
test "Can encode/decode float32 number field":
var proto = newProtoBuffer()
let num = float32(1234.164423)
var bytesProcessed: int
proto.encodeField(num)
var output = proto.output
assert output == @[13.byte, 67, 69, 154, 68]
var offset = 0
let decoded = decodeField(output, float32, offset, bytesProcessed)
assert decoded.value == num
assert decoded.index == 1
test "Can encode/decode float64 number field":
var proto = newProtoBuffer()
let num = 12343121537452.1644232341'f64
var bytesProcessed: int
proto.encodeField(num)
var output = proto.output
assert output == @[9.byte, 84, 88, 211, 191, 182, 115, 166, 66]
var offset = 0
let decoded = decodeField(output, float64, offset, bytesProcessed)
assert decoded.value == num
assert decoded.index == 1
test "Can encode/decode bool field":
var proto = newProtoBuffer()
let boolean = true
var bytesProcessed: int
proto.encodeField(boolean)
var output = proto.output
assert output == @[8.byte, 1]
var offset = 0
let decoded = decodeField(output, bool, offset, bytesProcessed)
assert bytesProcessed == 2
assert decoded.value == boolean
assert decoded.index == 1
test "Can encode/decode char field":
var proto = newProtoBuffer()
let charVal = 'G'
var bytesProcessed: int
proto.encodeField(charVal)
var output = proto.output
assert output == @[8.byte, ord(charVal).byte]
var offset = 0
let decoded = decodeField(output, char, offset, bytesProcessed)
assert bytesProcessed == 2
assert decoded.value == charVal
assert decoded.index == 1
test "Can encode/decode unsigned number field":
var proto = newProtoBuffer()
let num = 123151.uint
var bytesProcessed: int
proto.encodeField(num)
var output = proto.output
assert output == @[8.byte, 143, 194, 7]
var offset = 0
let decoded = decodeField(output, uint, offset, bytesProcessed)
assert decoded.value == num
assert decoded.index == 1
test "Can encode/decode string field":
var proto = newProtoBuffer()
let str = "hey this is a string"
var bytesProcessed: int
proto.encodeField(str)
var output = proto.output
assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103]
var offset = 0
let decoded = decodeField(output, string, offset, bytesProcessed)
assert decoded.value == str
assert decoded.index == 1
test "Can encode/decode char seq field":
var proto = newProtoBuffer()
let charSeq = "hey this is a string".toSeq
var bytesProcessed: int
proto.encodeField(charSeq)
var output = proto.output
assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103]
var offset = 0
let decoded = decodeField(output, seq[char], offset, bytesProcessed)
assert decoded.value == charSeq
assert decoded.index == 1
test "Can encode/decode uint8 seq field":
var proto = newProtoBuffer()
let uint8Seq = cast[seq[uint8]]("hey this is a string".toSeq)
var bytesProcessed: int
proto.encodeField(uint8Seq)
var output = proto.output
assert output == @[10.byte, 20, 104, 101, 121, 32, 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 115, 116, 114, 105, 110, 103]
var offset = 0
let decoded = decodeField(output, seq[uint8], offset, bytesProcessed)
assert decoded.value == uint8Seq
assert decoded.index == 1
test "Can encode/decode object field":
var proto = newProtoBuffer()
let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true, l: 124521.MyInt)
proto.encodeField(obj)
var offset, bytesProcessed: int
var output = proto.output
let decoded = decodeField(output, Test3, offset, bytesProcessed)
assert decoded.value == obj
assert decoded.index == 1
test "Can encode/decode object":
var proto = newProtoBuffer()
let obj = Test3(g: 300, h: 200, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true, l: 124521.MyInt)
proto.encode(obj)
var output = proto.output
let decoded = output.decode(Test3)
assert decoded == obj
test "Can encode/decode out of order object":
var proto = newProtoBuffer()
let obj = Test3(g: 400, h: 100, i: Test1(a: 100, b: "this is a test", c: 'H'), j: "testing", k: true, l: 14514.MyInt)
proto.encodeField(6, 14514.MyInt)
proto.encodeField(2, 100)
proto.encodeField(4, "testing")
proto.encodeField(1, 400)
proto.encodeField(3, Test1(a: 100, b: "this is a test", c: 'H'))
proto.encodeField(5, true)
var output = proto.output
let decoded = output.decode(Test3)
assert decoded == obj
test "Empty object field does not get encoded":
var proto = newProtoBuffer()
let obj = Test1()
proto.encodeField(1, obj)
var output = proto.output
assert output.len == 0
let decoded = output.decode(Test1)
assert decoded == obj
test "Empty object does not get encoded":
var proto = newProtoBuffer()
let obj = Test1()
proto.encode(obj)
var output = proto.output
assert output.len == 0
let decoded = output.decode(Test1)
assert decoded == obj

88
tests/test_stdlib.nim Normal file
View File

@ -0,0 +1,88 @@
import sets
import unittest
import ../protobuf_serialization
from ../protobuf_serialization/internal import unwrap
type
Basic = object
x {.pint, fieldNumber: 1.}: int32
y {.fieldNumber: 2.}: seq[string]
PragmadStdlib = object
x {.sint, fieldNumber: 1.}: seq[int32]
#y {.pint, fieldNumber: 2.}: array[5, uint32]
z {.pfloat32, fieldNumber: 3.}: HashSet[float32]
BooldStdlib = object
x {.fieldNumber: 1.}: seq[bool]
#y {.fieldNumber: 2.}: array[3, bool]
suite "Test Standard Lib Objects Encoding/Decoding":
#[test "Can encode/decode cstrings":
let str: cstring = "Testing string."
check Protobuf.decode(Protobuf.encode(str), type(cstring)) == str]#
test "Can encode/decode seqs":
let
int64Seq = @[SInt(0'i64), SInt(-1'i64), SInt(1'i64), SInt(-1'i64)]
read = Protobuf.decode(Protobuf.encode(int64Seq), seq[SInt(int64)])
check int64Seq.len == read.len
for i in 0 ..< int64Seq.len:
check int64Seq[i].unwrap() == read[i].unwrap()
let basicSeq = @[
Basic(
x: 0,
y: @[]
),
Basic(
x: 1,
y: @["abc", "defg"]
),
Basic(
x: 2,
y: @["hi", "jkl", "mnopq"]
),
Basic(
x: -2,
y: @["xyz"]
)
]
check basicSeq == Protobuf.decode(Protobuf.encode(basicSeq), seq[Basic])
#[test "Can encode/decode arrays":
let
int64Arr = [SInt(0'i64), SInt(-1'i64), SInt(1'i64), SInt(-1'i64)]
read = Protobuf.decode(Protobuf.encode(int64Arr), type(seq[SInt(int64)]))
check int64Arr.len == read.len
for i in 0 ..< int64Arr.len:
check int64Arr[i].unwrap() == read[i].unwrap()]#
test "Can encode/decode sets":
let
trueSet = {true}
falseSet = {false}
trueFalseSet = {true, false}
check Protobuf.decode(Protobuf.encode(trueSet), type(set[bool])) == trueSet
check Protobuf.decode(Protobuf.encode(falseSet), type(set[bool])) == falseSet
check Protobuf.decode(Protobuf.encode(trueFalseSet), type(set[bool])) == trueFalseSet
test "Can encode/decode HashSets":
let setInstance = ["abc", "def", "ghi"].toHashSet()
check Protobuf.decode(Protobuf.encode(setInstance), type(HashSet[string])) == setInstance
test "Can encode/decode stdlib fields where a pragma was used to specify encoding":
let pragmad = PragmadStdLib(
x: @[5'i32, -3'i32, 300'i32, -612'i32],
#y: [6'u32, 4'u32, 301'u32, 613'u32, 216'u32],
z: @[5.5'f32, 3.2'f32, 925.123].toHashSet()
)
check Protobuf.decode(Protobuf.encode(pragmad), PragmadStdLib) == pragmad
test "Can encode boolean seqs": #/arrays":
let boold = BooldStdlib(
x: @[true, false, true, true, false, false, false, true, false],
#y: [true, true, false]
)
check Protobuf.decode(Protobuf.encode(boold), BooldStdlib) == boold

77
tests/test_thirty_three_fields.nim Normal file
View File

@ -0,0 +1,77 @@
import unittest
import ../protobuf_serialization
type X = object
x00 {.fieldNumber: 1.}: bool
x01 {.fieldNumber: 2.}: bool
x02 {.fieldNumber: 3.}: bool
x03 {.fieldNumber: 4.}: bool
x04 {.fieldNumber: 5.}: bool
x05 {.fieldNumber: 6.}: bool
x06 {.fieldNumber: 7.}: bool
x07 {.fieldNumber: 8.}: bool
x08 {.fieldNumber: 9.}: bool
x09 {.fieldNumber: 10.}: bool
x0A {.fieldNumber: 11.}: bool
x0B {.fieldNumber: 12.}: bool
x0C {.fieldNumber: 13.}: bool
x0D {.fieldNumber: 14.}: bool
x0E {.fieldNumber: 15.}: bool
x0F {.fieldNumber: 16.}: bool
x10 {.fieldNumber: 17.}: bool
x11 {.fieldNumber: 18.}: bool
x12 {.fieldNumber: 19.}: bool
x13 {.fieldNumber: 20.}: bool
x14 {.fieldNumber: 21.}: bool
x15 {.fieldNumber: 22.}: bool
x16 {.fieldNumber: 23.}: bool
x17 {.fieldNumber: 24.}: bool
x18 {.fieldNumber: 25.}: bool
x19 {.fieldNumber: 26.}: bool
x1A {.fieldNumber: 27.}: bool
x1B {.fieldNumber: 28.}: bool
x1C {.fieldNumber: 29.}: bool
x1D {.fieldNumber: 30.}: bool
x1E {.fieldNumber: 31.}: bool
x1F {.fieldNumber: 32.}: bool
x20 {.fieldNumber: 33.}: bool
suite "Thirty-three fielded object":
test "Can encode and decode an object with 33 fields":
let x = X(
x00: true,
x01: true,
x02: true,
x03: true,
x04: true,
x05: true,
x06: true,
x07: true,
x08: true,
x09: true,
x0A: true,
x0B: true,
x0C: true,
x0D: true,
x0E: true,
x0F: true,
x10: true,
x11: true,
x12: true,
x13: true,
x14: true,
x15: true,
x16: true,
x17: true,
x18: true,
x19: true,
x1A: true,
x1B: true,
x1C: true,
x1D: true,
x1E: true,
x1F: true,
x20: true
)
check Protobuf.decode(Protobuf.encode(x), X) == x