From e8169c0ff4badd17cb4d47884c3cd8d12ac39e7f Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Tue, 10 Jan 2023 09:07:24 +0100 Subject: [PATCH] Cleanup / rewrite (#36) This is a cleanup / rewrite of the implementation increasing interoperability with other implementations and fixing several bugs / security issues * remove special handling of nim:isms (`ptr`, `ref`, `Option`, `HashSet` etc) (#4) * these deviate from "standard" protobuf behavior making the library less interoperable with other langagues * supporting "custom" types should be done as part of an extension framework instead (that can support any type / collection instead of a few hand-picked special cases) * don't allow encoding scalars in the core encoder (#31) * `codec` can be used to encode simple scalars * switch to `libp2p/minprotobuf`-like encoding base, fixing several bugs, crashes and inaccuracies (#30, #32, #33) * move parsing support to separate import * the parser is a heavy dependency * allow unknown fields * unknown fields should be given an extension point allowing the user to detect / handle them - standard behavior for protobuf is to ignore them * work around several faststreams bugs (#22) * remove machine-word-dependent length prefix options (#35) * actually, remove varint length prefix too for now (due to faststreams bugs) * update version * verify that strings are valid utf-8 on parse * fix warnings * truncate values like C++ version * allow unpacked fields in proto3 * protobuf2/3 -> proto2/3 * update docs There's lots left to do here in terms of tests and features: * Almost all tests are roundtrip tests - meaning they check that writing and reading have the same bugs (vs outputting conforming protobuf) * There are very few invalid-input tests * There's a beginning of an extension mechanism, but it needs more work * It's generally inefficient to copy data to a protobuf object and then write it to a stream - the stream writers should probably be made more general to handle this case better (either via callbacks or some other "builder-like" mechanism - projects currently using `minprotobuf` will likely see a performance regression using this library * `required` semantics are still off - a set/not-set flag is needed for every field in proto2 * possibly, when annotated with proto2, we should simply rewrite all members to become `PBOption` (as well as rename the field) --- README.md | 48 ++- protobuf_serialization.nim | 13 +- protobuf_serialization.nimble | 7 +- protobuf_serialization/codec.nim | 243 ++++++++++++ protobuf_serialization/files/proto_parser.nim | 2 +- .../files/type_generator.nim | 2 +- protobuf_serialization/internal.nim | 350 +++++------------ protobuf_serialization/numbers/common.nim | 87 ---- protobuf_serialization/numbers/fixed.nim | 85 ---- protobuf_serialization/numbers/varint.nim | 283 ------------- protobuf_serialization/pb_option.nim | 37 -- protobuf_serialization/proto_parser.nim | 2 + protobuf_serialization/reader.nim | 371 ++++++------------ protobuf_serialization/stdlib_readers.nim | 116 ------ protobuf_serialization/stdlib_writers.nim | 136 ------- protobuf_serialization/types.nim | 78 +++- protobuf_serialization/writer.nim | 333 ++++++---------- tests/files/test_proto3.nim | 14 +- tests/test_all.nim | 11 +- tests/test_bool.nim | 35 +- tests/test_codec.nim | 184 +++++++++ tests/test_different_types.nim | 31 -- tests/test_empty.nim | 49 +-- tests/test_fixed.nim | 37 +- tests/test_length_delimited.nim | 69 ---- tests/test_lint.nim | 52 --- tests/test_objects.nim | 261 ++---------- tests/test_options.nim | 139 ------- tests/test_protobuf2_semantics.nim | 65 ++- tests/test_repeated.nim | 60 +++ tests/test_repeated.proto | 14 + tests/test_stdlib.nim | 88 ----- tests/test_thirty_three_fields.nim | 4 +- 33 files changed, 1058 insertions(+), 2248 deletions(-) create mode 100644 protobuf_serialization/codec.nim delete mode 100644 protobuf_serialization/numbers/common.nim delete mode 100644 protobuf_serialization/numbers/fixed.nim delete mode 100644 protobuf_serialization/numbers/varint.nim delete mode 100644 protobuf_serialization/pb_option.nim create mode 100644 protobuf_serialization/proto_parser.nim delete mode 100644 protobuf_serialization/stdlib_readers.nim delete mode 100644 protobuf_serialization/stdlib_writers.nim create mode 100644 tests/test_codec.nim delete mode 100644 tests/test_different_types.nim delete mode 100644 tests/test_length_delimited.nim delete mode 100644 tests/test_lint.nim delete mode 100644 tests/test_options.nim create mode 100644 tests/test_repeated.nim create mode 100644 tests/test_repeated.proto delete mode 100644 tests/test_stdlib.nim diff --git a/README.md b/README.md index 77069e0..6b10d88 100644 --- a/README.md +++ b/README.md @@ -11,31 +11,47 @@ Protobuf implementation compatible with the [nim-serialization](https://github.c ## Usage -Due to the complexities of Protobuf, and the fact this library makes zero assumptions about encodings, extensive pragmas are required. +Messages in protobuf are serialized according to a schema found in `.proto` files. The library requires that types are annotated with schema information - this can be done either directly in Nim or, for some `proto3` files, generated using the `import_proto3` macro. -Both Protobuf 2 and Protobuf 3 semantics are supported. When declaring an object, add either the `protobuf2` or `protobuf3` pragma to declare which to use. When using Protobuf 3, a `import_proto3` macro is available. Taking in a file path, it can directly parse a Protobuf 3 spec file and generate the matching Nim types: +Both Protobuf 2 and Protobuf 3 semantics are supported. When declaring an object, add either the `proto2` or `proto3` pragma to declare which to use, as seen in the `syntax` element in protobuf. + +When using Protobuf 3, a `import_proto3` macro is available. Taking in a file path, it can directly parse a Protobuf 3 spec file and generate the matching Nim types, same as if they had been written manually. + +### Annotating objects + +The protobuf schema can be declared using annotations similar to what is found in a typical `.proto` file - see [types](./protobuf_serialization/types.nim) for available annotations: **my_protocol.proto3**: ```proto3 +syntax = "proto3"; + message ExampleMsg { int32 a = 1; float b = 2; } ``` -**nim_module.nim**: +**Annotated Nim code** ```nim +type ExampleMsg {.proto3.} = object + a {.fieldNumber: 1, pint.}: int32 + b {.fieldNumber: 2.}: float32 +``` + +**Importing proto file**: + +```nim +import protocol_serialization/proto_parser + +# This generates the same definition as above using a compile-time macro / parser import_proto3 "my_protocol.proto3" +``` -#[ -Generated: -type ExampleMsg {.protobuf3.} = object - a {.pint, fieldNumber: 1.}: int32 - b {.pfloat32, fieldNumber: 2.}: float32 -]# +**Encoding and decoding** +```nim let x = ExampleMsg(a: 10, b: 20.0) let encoded = Protobuf.encode(x) ... @@ -45,7 +61,7 @@ let decoded = Protobuf.decode(encoded, ExampleMsg) Both Protobuf 2 and Protobuf 3 objects have the following properties: - Every field requires the `fieldNumber` pragma, which takes in an integer of what field number to encode that field with. -- Every int/uint must have its bits explicitly specified. As the Nim compiler is unable to distinguish between a float with its bits explicitly specified and a float, `pfloat32` or `pfloat64` is required. +- Every int/uint must have its bits explicitly specified. - int/uint fields require their encoding to be specified. `pint` is valid for both, and uses VarInt encoding, which only uses the amount of bytes it needs. `fixed` is also valid for both, and uses the full amount of bytes the number uses, instead of stripping unused bytes. This has performance advantages for large numbers. Finally, `sint` uses zig-zagged VarInt encoding, which is recommended for numbers which are frequently negative, and is only valid for ints. Protobuf 2 has the additional properties: @@ -57,14 +73,14 @@ Here is an example demonstrating how the various pragmas can be combined: ```nim type - X {.protobuf3.} = object - a {.pint, fieldNumber: 1.}: int32 - b {.pfloat32, fieldNumber: 2.}: float32 + X {.proto3.} = object + a {.fieldNumber: 1, pint.}: int32 + b {.fieldNumber: 2.}: float32 - Y {.protobuf2.} = object + Y {.proto2.} = object a {.fieldNumber: 1.}: seq[string] - b {.pint, fieldNumber: 2.}: PBOption[int32(2)] - c {.required, sint, fieldNumber: 3.}: int32 + b {.fieldNumber: 2, pint.}: PBOption[int32(2)] + c {.fieldNumber: 3, required, sint.}: int32 ``` ## License diff --git a/protobuf_serialization.nim b/protobuf_serialization.nim index b24cdd9..20106fb 100644 --- a/protobuf_serialization.nim +++ b/protobuf_serialization.nim @@ -6,9 +6,6 @@ export serialization import protobuf_serialization/[internal, types, reader, writer] export types, reader, writer -import protobuf_serialization/files/type_generator -export protoToTypes, import_proto3 - serializationFormat Protobuf Protobuf.setReader ProtobufReader @@ -20,17 +17,13 @@ func supportsInternal[T](ty: typedesc[T], handled: var HashSet[string]) {.compil handled.incl($T) verifySerializable(T) - var inst: T - enumInstanceSerializedFields(inst, fieldName, fieldVar): - discard fieldName - when flatType(fieldVar) is (object or tuple): - supportsInternal(flatType(fieldVar), handled) func supportsCompileTime[T](_: typedesc[T]) = - when flatType(T) is (object or tuple): + when flatType(default(T)) is (object or tuple): var handled = initHashSet[string]() - supportsInternal(flatType(T), handled) + supportsInternal(flatType(default(T)), handled) func supports*[T](_: type Protobuf, ty: typedesc[T]): bool = + # TODO return false when not supporting, instead of crashing compiler static: supportsCompileTime(T) true diff --git a/protobuf_serialization.nimble b/protobuf_serialization.nimble index f7a8602..ed95ba5 100644 --- a/protobuf_serialization.nimble +++ b/protobuf_serialization.nimble @@ -2,8 +2,8 @@ import os, strutils mode = ScriptMode.Verbose -version = "0.2.0" -author = "Joey Yakimowich-Payne" +version = "0.3.0" +author = "Status" description = "Protobuf implementation compatible with the nim-serialization framework." license = "MIT" skipDirs = @["tests"] @@ -12,7 +12,8 @@ requires "nim >= 1.2.0", "stew", "faststreams", "serialization", - "combparser" + "combparser", + "unittest2" const styleCheckStyle = if (NimMajor, NimMinor) < (1, 6): diff --git a/protobuf_serialization/codec.nim b/protobuf_serialization/codec.nim new file mode 100644 index 0000000..43d953a --- /dev/null +++ b/protobuf_serialization/codec.nim @@ -0,0 +1,243 @@ +# Copyright (c) 2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +## This module implements core primitives for the protobuf language as seen in +## `.proto` files + +# TODO fix exception raising - should probably only raise ProtoError derivatives +# and whatever streams already raises +# +# when (NimMajor, NimMinor) < (1, 4): +# {.push raises: [Defect].} +# else: +# {.push raises: [].} + +import + std/[typetraits, unicode], + faststreams, + stew/[leb128, endians2] + +type + WireKind* = enum + Varint = 0 + Fixed64 = 1 + LengthDelim = 2 + # StartGroup = 3 # Not used + # EndGroup = 4 # Not used + Fixed32 = 5 + + FieldHeader* = distinct uint32 + + # Scalar types used in `.proto` files + # https://developers.google.com/protocol-buffers/docs/proto3#scalar + pdouble* = distinct float64 + pfloat* = distinct float32 + + pint32* = distinct int32 ## varint-encoded signed integer + pint64* = distinct int64 ## varint-encoded signed integer + + puint32* = distinct uint32 ## varint-encoded unsigned integer + puint64* = distinct uint64 ## varint-encoded unsigned integer + + sint32* = distinct int32 ## zig-zag-varint-encoded signed integer + sint64* = distinct int64 ## zig-zag-varint-encoded signed integer + + fixed32* = distinct uint32 ## fixed-width unsigned integer + fixed64* = distinct uint64 ## fixed-width unsigned integer + + sfixed32* = distinct int32 ## fixed-width signed integer + sfixed64* = distinct int64 ## fixed-width signed integer + + pbool* = distinct bool + + pstring* = distinct string ## UTF-8-encoded string + pbytes* = distinct seq[byte] ## byte sequence + + SomeScalar* = + pint32 | pint64 | puint32 | puint64 | sint32 | sint64 | pbool | + fixed64 | sfixed64 | pdouble | + pstring | pbytes | + fixed32 | sfixed32 | pfloat + + # Mappings of proto type to wire type + SomeVarint* = + pint32 | pint64 | puint32 | puint64 | sint32 | sint64 | pbool + SomeFixed64* = fixed64 | sfixed64 | pdouble + SomeLengthDelim* = pstring | pbytes # Also messages and packed repeated fields + SomeFixed32* = fixed32 | sfixed32 | pfloat + + SomePrimitive* = SomeVarint | SomeFixed64 | SomeFixed32 + ## Types that may appear packed + +const + SupportedWireKinds* = { + uint8(WireKind.Varint), + uint8(WireKind.Fixed64), + uint8(WireKind.LengthDelim), + uint8(WireKind.Fixed32) + } + +template wireKind*(T: type SomeVarint): WireKind = WireKind.Varint +template wireKind*(T: type SomeFixed64): WireKind = WireKind.Fixed64 +template wireKind*(T: type SomeLengthDelim): WireKind = WireKind.LengthDelim +template wireKind*(T: type SomeFixed32): WireKind = WireKind.Fixed32 + +template validFieldNumber*(i: int, strict: bool = false): bool = + # https://developers.google.com/protocol-buffers/docs/proto#assigning + # Field numbers in the 19k range are reserved for the protobuf implementation + (i > 0 and i < (1 shl 29)) and (not strict or not(i >= 19000 and i <= 19999)) + +template init*(_: type FieldHeader, index: int, wire: WireKind): FieldHeader = + ## Get protobuf's field header integer for ``index`` and ``wire``. + FieldHeader((uint32(index) shl 3) or uint32(wire)) + +template number*(p: FieldHeader): int = + int(uint32(p) shr 3) + +template kind*(p: FieldHeader): WireKind = + cast[WireKind](uint8(p) and 0x07'u8) # 3 lower bits + +template toUleb(x: puint64): uint64 = uint64(x) +template toUleb(x: puint32): uint32 = uint32(x) + +func toUleb(x: sint64): uint64 = + let v = cast[uint64](x) + (v shl 1) xor (0 - (v shr 63)) + +func toUleb(x: sint32): uint32 = + let v = cast[uint32](x) + (v shl 1) xor (0 - (v shr 31)) + +template toUleb(x: pint64): uint64 = cast[uint64](x) +template toUleb(x: pint32): uint32 = cast[uint32](x) +template toUleb(x: pbool): uint8 = cast[uint8](x) + +template fromUleb(x: uint64, T: type puint64): T = puint64(x) +template fromUleb(x: uint64, T: type pbool): T = pbool(x != 0) + +template fromUleb(x: uint64, T: type puint64): T = puint64(x) +template fromUleb(x: uint64, T: type puint32): T = puint32(x) + +template fromUleb(x: uint64, T: type sint64): T = + cast[T]((x shr 1) xor (0 - (x and 1))) +template fromUleb(x: uint64, T: type sint32): T = + cast[T]((uint32(x) shr 1) xor (0 - (uint32(x) and 1))) + +template fromUleb(x: uint64, T: type pint64): T = cast[T](x) +template fromUleb(x: uint64, T: type pint32): T = cast[T](x) + +template toBytes*(x: SomeVarint): openArray[byte] = + toBytes(toUleb(x), Leb128).toOpenArray() + +template toBytes*(x: fixed32 | fixed64): openArray[byte] = + type Base = distinctBase(typeof(x)) + toBytesLE(Base(x)) + +template toBytes*(x: sfixed32): openArray[byte] = + toBytes(fixed32(x)) +template toBytes*(x: sfixed64): openArray[byte] = + toBytes(fixed64(x)) + +template toBytes*(x: pdouble): openArray[byte] = + cast[array[8, byte]](x) +template toBytes*(x: pfloat): openArray[byte] = + cast[array[4, byte]](x) + +template toBytes*(header: FieldHeader): openArray[byte] = + toBytes(uint32(header), Leb128).toOpenArray() + +proc vsizeof*(x: SomeVarint): int = + ## Returns number of bytes required to encode integer ``x`` as varint. + Leb128.len(toUleb(x)) + +proc writeValue*(output: OutputStream, value: SomeVarint) = + output.write(toBytes(value)) + +proc writeValue*(output: OutputStream, value: SomeFixed64) = + output.write(toBytes(value)) + +proc writeValue*(output: OutputStream, value: pstring) = + output.write(toBytes(puint64(string(value).len()))) + output.write(string(value).toOpenArrayByte(0, string(value).high())) + +proc writeValue*(output: OutputStream, value: pbytes) = + output.write(toBytes(puint64(seq[byte](value).len()))) + output.write(seq[byte](value)) + +proc writeValue*(output: OutputStream, value: SomeFixed32) = + output.write(toBytes(value)) + +proc writeField*(output: OutputStream, field: int, value: SomeScalar) = + output.write(toBytes(FieldHeader.init(field, wireKind(typeof(value))))) + output.writeValue(value) + +proc readValue*[T: SomeVarint](input: InputStream, _: type T): T = + # TODO This is not entirely correct: we should truncate value if it doesn't + # fit, according to the docs: + # https://developers.google.com/protocol-buffers/docs/proto#updating + var buf: Leb128Buf[uint64] + while buf.len < buf.data.len and input.readable(): + let b = input.read() + buf.data[buf.len] = b + buf.len += 1 + if (b and 0x80'u8) == 0: + break + + let (val, len) = uint64.fromBytes(buf) + if buf.len == 0 or len != buf.len: + raise (ref ValueError)(msg: "Cannot read varint from stream") + + fromUleb(val, T) + +proc readValue*[T: SomeFixed32 | SomeFixed64](input: InputStream, _: type T): T = + var tmp {.noinit.}: array[sizeof(T), byte] + if not input.readInto(tmp): + raise (ref ValueError)(msg: "Not enough bytes") + when T is pdouble | pfloat: + copyMem(addr result, addr tmp[0], sizeof(result)) + elif sizeof(T) == 8: + cast[T](uint64.fromBytesLE(tmp)) # Cast so we don't run into signed trouble + else: + cast[T](uint32.fromBytesLE(tmp)) # Cast so we don't run into signed trouble + +proc readLength*(input: InputStream): int = + let lenu32 = input.readValue(puint32) + if uint64(lenu32) > uint64(int.high()): + raise (ref ValueError)(msg: "Invalid length") + int(lenu32) + +proc readValue*[T: SomeLengthDelim](input: InputStream, _: type T): T = + let len = input.readLength() + if len > 0: + type Base = typetraits.distinctBase(T) + let inputLen = input.len() + if inputLen.isSome() and len > inputLen.get(): + raise (ref ValueError)(msg: "Missing bytes: " & $len) + + Base(result).setLen(len) + template bytes(): openArray[byte] = + when Base is seq[byte]: + Base(result).toOpenArray(0, len - 1) + else: + Base(result).toOpenArrayByte(0, len - 1) + if not input.readInto(bytes()): + raise (ref ValueError)(msg: "Missing bytes: " & $len) + + when T is pstring: + if validateUtf8(string(result)) != -1: + raise (ref ValueError)(msg: "String not valid UTF-8") + +proc readHeader*(input: InputStream): FieldHeader = + let + hdr = uint32(input.readValue(puint32)) + wire = uint8(hdr and 0x07) + + if wire notin SupportedWireKinds: + raise (ref ValueError)(msg: "Invalid wire type") + + FieldHeader(hdr) diff --git a/protobuf_serialization/files/proto_parser.nim b/protobuf_serialization/files/proto_parser.nim index 211773e..1d739ab 100644 --- a/protobuf_serialization/files/proto_parser.nim +++ b/protobuf_serialization/files/proto_parser.nim @@ -206,7 +206,7 @@ proc protofile*(): StringParser[ProtoNode] = (syntaxline() + optional(package()) result.imported.add message of Enum: result.package.packageEnums.add message - else: raise newException(AssertionError, "Unsupported node kind: " & $message.kind) + else: raiseAssert "Unsupported node kind: " & $message.kind ) macro expandToFullDef(protoParsed: var ProtoNode, filepath: string, stringGetter: untyped): untyped = result = quote do: diff --git a/protobuf_serialization/files/type_generator.nim b/protobuf_serialization/files/type_generator.nim index 052e871..6a782ab 100644 --- a/protobuf_serialization/files/type_generator.nim +++ b/protobuf_serialization/files/type_generator.nim @@ -121,7 +121,7 @@ proc protoToTypesInternal*(filepath: string, proto: string): NimNode = newNimNode(nnkTypeDef).add( newNimNode(nnkPragmaExpr).add( newNimNode(nnkPostfix).add(ident("*"), ident(name)), - newNimNode(nnkPragma).add(ident("protobuf3")) + newNimNode(nnkPragma).add(ident("proto3")) ), newEmptyNode(), value diff --git a/protobuf_serialization/internal.nim b/protobuf_serialization/internal.nim index 42b7ac2..37f5ec7 100644 --- a/protobuf_serialization/internal.nim +++ b/protobuf_serialization/internal.nim @@ -1,296 +1,142 @@ #Variables needed by the Reader and Writer which should NOT be exported outside of this library. -import options -import sets -import tables - +import std/[options, sets] import stew/shims/macros #Depending on the situation, one of these two are used. #Sometimes, one works where the other doesn't. #It all comes down to bugs in Nim and managing them. export getCustomPragmaVal, getCustomPragmaFixed export hasCustomPragmaFixed + import serialization -import numbers/varint -import numbers/fixed -export varint -export fixed +import "."/[codec, types] -import pb_option -export pb_option - -const WIRE_TYPE_MASK = 0b0000_0111'i32 - -type - ProtobufWireType* = enum - VarInt, Fixed64, LengthDelimited, StartGroup, EndGroup, Fixed32 - - ProtobufKey* = object - number*: int - wire*: ProtobufWireType - - #Number types which are platform-dependent and therefore unsafe. - PlatformDependentTypes* = int or uint - - #Castable length delimited types. - #These can be directly casted from a seq[byte] and do not require a custom converter. - CastableLengthDelimitedTypes* = seq[byte or char or bool] - #This type is literally every other type. - #Every other type is considered custom, due to the need for their own converters. - #While cstring/array are built-ins, and therefore should have converters provided, but they still need converters. - LengthDelimitedTypes* = not (VarIntTypes or FixedTypes) - - #Disabled types. - Disabled = LUIntWrapped or array or cstring or tuple or Table - -template isPotentiallyNull*[T](ty: typedesc[T]): bool = - T is (Option or PBOption or ref or ptr) - -template getUnderlyingType*[I]( - stdlib: seq[I] or set[I] or HashSet[I] -): untyped = - I +type UnsupportedType*[FieldType; RootType; fieldName: static string] = object proc flatTypeInternal(value: auto): auto {.compileTime.} = - when value is (Option or PBOption): + when value is PBOption: flatTypeInternal(value.get()) - elif value is (ref or ptr): - flatTypeInternal(value[]) else: value template flatType*(value: auto): type = type(flatTypeInternal(value)) -template flatType*[B](ty: typedesc[B]): type = - when B is openArray: - B - else: - var blank: B - flatType(blank) +macro unsupportedProtoType*(FieldType, RootType, fieldName: typed): untyped = + # TODO turn this into an extension point + # TODO fix RootType printing + error "Serializing " & humaneTypeName(FieldType) & " as field type is not supported: " & humaneTypeName(RootType) & "." & repr(fieldName) -proc flatMapInternal[B, T](value: B, ty: typedesc[T]): Option[T] = - when value is (Option or PBOption): - if value.isNone(): - return - flatMapInternal(value.get(), ty) - elif value is (ref or ptr): - if value.isNil(): - return - flatMapInternal(value[], ty) - else: - some(value) +proc isProto2*(T: type): bool {.compileTime.} = T.hasCustomPragma(proto2) +proc isProto3*(T: type): bool {.compileTime.} = T.hasCustomPragma(proto3) -template flatMap*(value: auto): auto = - flatMapInternal(value, flatType(value)) - -func isStdlib*[B](_: typedesc[B]): bool {.compileTime.} = - flatType(B) is (cstring or string or seq or array or set or HashSet) - -func mustUseSingleBuffer*[T](_: typedesc[T]): bool {.compileTime.} - -func convertAndCallMustUseSingleBuffer[T]( - _: typedesc[seq[T] or openArray[T] or set[T] or HashSet[T]] -): bool {.compileTime.} = - when flatType(T).isStdlib(): - false - else: - mustUseSingleBuffer(flatType(T)) - -#[func convertAndCallMustUseSingleBuffer[C, T]( - _: typedesc[array[C, T]] -): bool {.compileTime.} = - when flatType(T).isStdlib(): - false - else: - mustUseSingleBuffer(flatType(T))]# - -func mustUseSingleBuffer*[T](_: typedesc[T]): bool {.compileTime.} = - when flatType(T) is (cstring or string or seq[byte or char or bool]): - true - elif flatType(T) is (array or openArray or set or HashSet): - flatType(T).convertAndCallMustUseSingleBuffer() - else: - false - -func singleBufferable*[T](_: typedesc[T]): bool {.compileTime.} - -func convertAndCallSingleBufferable[T]( - _: typedesc[seq[T] or openArray[T] or set[T] or HashSet[T]] -): bool {.compileTime.} = - when flatType(T).isStdlib(): - false - else: - singleBufferable(flatType(T)) - -#[func convertAndCallSingleBufferable[C, T]( - _: typedesc[array[C, T]] -): bool {.compileTime.} = - when flatType(T).isStdlib(): - false - else: - singleBufferable(flatType(T))]# - -func singleBufferable*[T](_: typedesc[T]): bool {.compileTime.} = - when flatType(T).mustUseSingleBuffer(): - true - elif flatType(T) is (VarIntTypes or FixedTypes): - true - elif flatType(T) is (seq or array or openArray or set or HashSet): - flatType(T).convertAndCallSingleBufferable() - else: - false - -template nextType[B](box: B): auto = - when B is (Option or PBOption): - box.get() - elif B is (ref or ptr): - box[] - else: - box - -proc boxInternal[C, B](value: C, into: B): B = - when value is B: - value - elif into is (Option or PBOption): - var blank: type(nextType(into)) - #We never access this pointer. - #Ever. - #That said, in order for this Option to resolve as some, it can't be nil. - when blank is ref: - blank = cast[type(blank)](1) - elif blank is ptr: - blank = cast[type(blank)](1) - let temp = some(blank) - when into is Option: - some(boxInternal(value, nextType(temp))) +proc isPacked*(T: type, fieldName: static string): Option[bool] {.compileTime.} = + if T.hasCustomPragmaFixed(fieldName, packed): + const p = T.getCustomPragmaFixed(fieldName, packed) + when p is NimNode: + none(bool) else: - pbSome(type(into), boxInternal(value, nextType(temp))) - elif into is ref: - new(result) - result[] = boxInternal(value, nextType(result)) - elif into is ptr: - result = cast[B](alloc0(sizeof(B))) - result[] = boxInternal(value, nextType(result)) + some(p) + else: + none(bool) -proc box*[B](into: var B, value: auto) = - into = boxInternal(value, into) +proc isRequired*(T: type, fieldName: static string): bool {.compileTime.} = + T.hasCustomPragmaFixed(fieldName, required) -template protobuf2*() {.pragma.} -template protobuf3*() {.pragma.} -template fieldNumber*(num: int) {.pragma.} -template required*() {.pragma.} +proc fieldNumberOf*(T: type, fieldName: static string): int {.compileTime.} = + T.getCustomPragmaFixed(fieldName, fieldNumber) + +template protoType*(InnerType, RootType, FieldType: untyped, fieldName: untyped) = + mixin flatType + when FieldType is seq and FieldType isnot seq[byte]: + type FlatType = flatType(default(typeof(for a in default(FieldType): a))) + else: + type FlatType = flatType(default(FieldType)) + when FlatType is float64: + type InnerType = pdouble + elif FlatType is float32: + type InnerType = pfloat + elif FlatType is int32: + when RootType.hasCustomPragmaFixed(fieldName, pint): + type InnerType = pint32 + elif RootType.hasCustomPragmaFixed(fieldName, sint): + type InnerType = sint32 + elif RootType.hasCustomPragmaFixed(fieldName, fixed): + type InnerType = sfixed32 + else: + {.fatal: "Must annotate `int32` fields with `pint`, `sint` or `fixed`".} + elif FlatType is int64: + when RootType.hasCustomPragmaFixed(fieldName, pint): + type InnerType = pint64 + elif RootType.hasCustomPragmaFixed(fieldName, sint): + type InnerType = sint64 + elif RootType.hasCustomPragmaFixed(fieldName, fixed): + type InnerType = sfixed64 + else: + {.fatal: "Must annotate `int64` fields with `pint`, `sint` or `fixed`".} + elif FlatType is uint32: + when RootType.hasCustomPragmaFixed(fieldName, fixed): + type InnerType = fixed32 + else: + type InnerType = puint32 + elif FlatType is uint64: + when RootType.hasCustomPragmaFixed(fieldName, fixed): + type InnerType = fixed64 + else: + type InnerType = puint64 + elif FlatType is bool: + type InnerType = pbool + elif FlatType is string: + type InnerType = pstring + elif FlatType is seq[byte]: + type InnerType = pbytes + elif FlatType is object: + type InnerType = FieldType + else: + type InnerType = UnsupportedType[FieldType, RootType, fieldName] + +template elementType[T](_: type seq[T]): type = typeof(T) -#Created in response to https://github.com/kayabaNerve/nim-protobuf-serialization/issues/5. func verifySerializable*[T](ty: typedesc[T]) {.compileTime.} = - when T is PlatformDependentTypes: + type FlatType = flatType(default(T)) + when FlatType is int | uint: {.fatal: "Serializing a number requires specifying the amount of bits via the type.".} - elif T is SomeFloat: - {.fatal: "Couldnt serialize the float; all floats need their bits specified with a Float32 or Float64 call.".} - elif T is PureTypes: - {.fatal: "Serializing a number requires specifying the encoding to use.".} - #LUIntWrapped is disabled; provide a better error. - elif T is LUIntWrapped: - {.fatal: "LibP2P VarInts are only usable directly with encodeVarInt.".} - elif T is Disabled: - const DISABLED_STRING = "Arrays, cstrings, tuples, and Tables are not serializable due to various reasons." - {.fatal: DISABLED_STRING.} - elif T.isStdlib(): - discard - #Tuple inclusion is so in case we can add back support for tuples, we solely have to delete the above case. - elif T is (object or tuple): + elif FlatType is seq: + verifySerializable(elementType(T)) + elif FlatType is object: var inst: T fieldNumberSet = initHashSet[int]() discard fieldNumberSet const - pb2: bool = T.hasCustomPragma(protobuf2) - pb3: bool = T.hasCustomPragma(protobuf3) - when pb2 == pb3: - {.fatal: "Serialized objects must have either the protobuf2 or protobuf3 pragma attached.".} + isProto2 = T.isProto2() + isProto3 = T.isProto3() + when isProto2 == isProto3: + {.fatal: "Serialized objects must have either the proto2 or proto3 pragma attached.".} enumInstanceSerializedFields(inst, fieldName, fieldVar): - when pb2 and (not ty.hasCustomPragmaFixed(fieldName, required)): - when fieldVar is not (seq or set or HashSet): - when fieldVar is not (Option or PBOption): - {.fatal: "Protobuf2 requires every field to either have the required pragma attached or be a repeated field/PBOption/Option.".} - when pb3 and ( - ty.hasCustomPragmaFixed(fieldName, required) or + when isProto2 and not T.isRequired(fieldName): + when fieldVar is not seq: + when fieldVar is not PBOption: + {.fatal: "proto2 requires every field to either have the required pragma attached or be a repeated field/PBOption.".} + when isProto3 and ( + T.hasCustomPragmaFixed(fieldName, required) or (fieldVar is PBOption) ): - {.fatal: "The required pragma/PBOption type can only be used with Protobuf2.".} + {.fatal: "The required pragma/PBOption type can only be used with proto2.".} - when fieldVar is PlatformDependentTypes: - {.fatal: "Serializing a number requires specifying the amount of bits via the type.".} - elif T is LUIntWrapped: - {.fatal: "LibP2P VarInts are only usable directly with encodeVarInt.".} - elif T is Disabled: - {.fatal: DISABLED_STRING & " are not serializable due to various reasons.".} - elif fieldVar is (VarIntTypes or FixedTypes): - const - hasPInt = ty.hasCustomPragmaFixed(fieldName, pint) - hasSInt = ty.hasCustomPragmaFixed(fieldName, sint) - hasFixed = ty.hasCustomPragmaFixed(fieldName, fixed) - when fieldVar is (VarIntWrapped or FixedWrapped): - when uint(hasPInt) + uint(hasSInt) + uint(hasFixed) != 0: - {.fatal: "Encoding specified for an already wrapped type, or a type which isn't wrappable due to always having one encoding (byte, char, bool, or float).".} + protoType(ProtoType {.used.}, T, typeof(fieldVar), fieldName) # Ensure we can form a ProtoType - when fieldVar is SomeFloat: - const - hasF32 = ty.hasCustomPragmaFixed(fieldName, pfloat32) - hasF64 = ty.hasCustomPragmaFixed(fieldName, pfloat64) - when hasF32: - when sizeof(fieldVar) != 4: - {.fatal: "pfloat32 pragma attached to a 64-bit float.".} - elif hasF64: - when sizeof(fieldVar) != 8: - {.fatal: "pfloat64 pragma attached to a 32-bit float.".} - else: - {.fatal: "Floats require the pfloat32 or pfloat64 pragma to be attached.".} - elif uint(hasPInt) + uint(hasSInt) + uint(hasFixed) != 1: - {.fatal: "Couldn't write " & fieldName & "; either none or multiple encodings were specified.".} - - const thisFieldNumber = fieldVar.getCustomPragmaVal(fieldNumber) - when thisFieldNumber is NimNode: + const fieldNum = T.fieldNumberOf(fieldName) + when fieldNum is NimNode: {.fatal: "No field number specified on serialized field.".} else: - when thisFieldNumber <= 0: - {.fatal: "Negative field number or 0 field number was specified. Protobuf fields start at 1.".} - elif thisFieldNumber shr 28 != 0: - #I mean, it is technically serializable with an uint64 (max 2^60), or even uint32 (max 2^29). - #That said, having more than 2^28 fields should never be needed. Why lose performance for a never-useful case? - {.fatal: "Field number greater than 2^28 specified. On 32-bit systems, this isn't serializable.".} + when not validFieldNumber(fieldNum, strict = true): + {.fatal: "Field numbers must be in the range [1..2^29-1]".} - if fieldNumberSet.contains(thisFieldNumber): - raise newException(Exception, "Field number was used twice on two different fields.") - fieldNumberSet.incl(thisFieldNumber) + if fieldNumberSet.containsOrIncl(fieldNum): + raiseAssert "Field number was used twice on two different fields: " & $fieldNum -proc newProtobufKey*(number: int, wire: ProtobufWireType): seq[byte] = - result = newSeq[byte](5) - var viLen = 0 - doAssert encodeVarInt( - result, - viLen, - PInt((int32(number) shl 3) or int32(wire)) - ) == VarIntStatus.Success - result.setLen(viLen) - -proc writeProtobufKey*( - stream: OutputStream, - number: int, - wire: ProtobufWireType -) {.inline.} = - stream.write(newProtobufKey(number, wire)) - -proc readProtobufKey*( - stream: InputStream -): ProtobufKey = - let - varint = stream.decodeVarInt(int, PInt(int32)) - wire = byte(varint and WIRE_TYPE_MASK) - if (wire < byte(low(ProtobufWireType))) or (byte(high(ProtobufWireType)) < wire): - raise newException(ProtobufMessageError, "Protobuf key had an invalid wire type.") - result.wire = ProtobufWireType(wire) - result.number = varint shr 3 + # verifySerializable(typeof(fieldVar)) diff --git a/protobuf_serialization/numbers/common.nim b/protobuf_serialization/numbers/common.nim deleted file mode 100644 index 4fb479b..0000000 --- a/protobuf_serialization/numbers/common.nim +++ /dev/null @@ -1,87 +0,0 @@ -import macros - -import serialization - -type - #Defined here so the number encoders/decoders have access. - ProtobufError* = object of SerializationError - - ProtobufReadError* = object of ProtobufError - ProtobufEOFError* = object of ProtobufReadError - ProtobufMessageError* = object of ProtobufReadError - - #Signed native types. - PureSIntegerTypes* = SomeSignedInt or enum - #Unsigned native types. - PureUIntegerTypes* = SomeUnsignedInt or char or bool - #Every native type. - PureTypes* = (PureSIntegerTypes or PureUIntegerTypes) and - (not (byte or char or bool)) - -macro generateWrapper*( - name: untyped, - supported: typed, - exclusion: typed, - uTypes: typed, - uLarger: typed, - uSmaller: typed, - sTypes: typed, - sLarger: typed, - sSmaller: typed, - err: string -): untyped = - let strLitName = newStrLitNode(name.strVal) - quote do: - template `name`*(value: untyped): untyped = - when (value is (bool or byte or char)) and (`strLitName` != "PInt"): - {.fatal: "Byte types are always PInt.".} - - #If this enum doesn't have negative values, considered it unsigned. - when value is enum: - when value is type: - when ord(low(value)) < 0: - type fauxType = int32 - else: - type fauxType = uint32 - else: - when ord(low(type(value))) < 0: - type fauxType = int32 - else: - type fauxType = uint32 - elif value is type: - type fauxType = value - else: - type fauxType = type(value) - - when fauxType is not `supported`: - {.fatal: `err`.} - elif fauxType is `exclusion`: - {.fatal: "Tried to rewrap a wrapped type.".} - - when value is type: - when fauxType is `uTypes`: - when sizeof(value) == 8: - `uLarger` - else: - `uSmaller` - elif fauxType is `sTypes`: - when sizeof(value) == 8: - `sLarger` - else: - `sSmaller` - #Used for Fixed floats. - else: - value - else: - when fauxType is `uTypes`: - when sizeof(value) == 8: - `uLarger`(value) - else: - `uSmaller`(value) - elif fauxType is `sTypes`: - when sizeof(value) == 8: - `sLarger`(value) - else: - `sSmaller`(value) - else: - value diff --git a/protobuf_serialization/numbers/fixed.nim b/protobuf_serialization/numbers/fixed.nim deleted file mode 100644 index e5447a4..0000000 --- a/protobuf_serialization/numbers/fixed.nim +++ /dev/null @@ -1,85 +0,0 @@ -import faststreams - -import common -export PureTypes -export ProtobufError, ProtobufReadError, ProtobufEOFError, ProtobufMessageError - -const LAST_BYTE* = 0b1111_1111 - -type - FixedWrapped64 = distinct uint64 - FixedWrapped32 = distinct uint32 - SFixedWrapped64 = distinct int64 - SFixedWrapped32 = distinct int32 - FloatWrapped64 = distinct uint64 - FloatWrapped32 = distinct uint32 - - FixedDistinctWrapped = FixedWrapped64 or FixedWrapped32 or - SFixedWrapped64 or SFixedWrapped32 or - FloatWrapped64 or FloatWrapped32 - - FixedWrapped* = FixedDistinctWrapped or float64 or float32 - - WrappableFixedTypes = PureUIntegerTypes or PureSIntegerTypes - - #Every type valid for the Fixed (64 or 43) wire type. - FixedTypes* = FixedWrapped or WrappableFixedTypes - -generateWrapper( - Fixed, WrappableFixedTypes, FixedDistinctWrapped, - PureUIntegerTypes, FixedWrapped64, FixedWrapped32, - PureSIntegerTypes, SFixedWrapped64, SFixedWrapped32, - "Fixed should only be used with a non-float number. Floats are always fixed already." -) - -template Float64*(value: float64): FloatWrapped64 = - cast[FloatWrapped64](value) - -template Float32*(value: float32): FloatWrapped32 = - cast[FloatWrapped32](value) - -template unwrap*(value: FixedWrapped): untyped = - when value is FixedWrapped64: - uint64(value) - elif value is FixedWrapped32: - uint32(value) - elif value is SFixedWrapped64: - int64(value) - elif value is SFixedWrapped32: - int32(value) - elif value is FloatWrapped64: - float64(value) - elif value is FloatWrapped32: - float32(value) - elif value is (float64 or float32): - value - else: - {.fatal: "Tried to get the unwrapped value of a non-wrapped type. This should never happen.".} - -template fixed*() {.pragma.} -template pfloat32*() {.pragma.} -template pfloat64*() {.pragma.} - -proc encodeFixed*(stream: OutputStream, value: FixedWrapped) = - when sizeof(value) == 8: - var casted = cast[uint64](value) - else: - var casted = cast[uint32](value) - - for _ in 0 ..< sizeof(casted): - stream.write(byte(casted and LAST_BYTE)) - casted = casted shr 8 - -proc decodeFixed*( - stream: InputStream, - res: var auto -) = - when sizeof(res) == 8: - var temp: uint64 - else: - var temp: uint32 - for i in 0 ..< sizeof(temp): - if not stream.readable(): - raise newException(ProtobufEOFError, "Stream ended before the Fixed number was finished.") - temp = temp + (type(temp)(stream.read()) shl (i * 8)) - res = cast[type(res)](temp) diff --git a/protobuf_serialization/numbers/varint.nim b/protobuf_serialization/numbers/varint.nim deleted file mode 100644 index c4271a7..0000000 --- a/protobuf_serialization/numbers/varint.nim +++ /dev/null @@ -1,283 +0,0 @@ -import stew/bitops2 -import faststreams - -import common -export PureTypes -export ProtobufError, ProtobufReadError, ProtobufEOFError, ProtobufMessageError - -const - VAR_INT_CONTINUATION_MASK*: byte = 0b1000_0000 - VAR_INT_VALUE_MASK: byte = 0b0111_1111 - -type - VarIntStatus* = enum - Success, - Overflow, - Incomplete - - #Used to specify how to encode/decode primitives. - #Despite being used outside of this library, all access is via templates. - PIntWrapped32 = distinct int32 - PIntWrapped64 = distinct int64 - SIntWrapped32 = distinct int32 - SIntWrapped64 = distinct int64 - UIntWrapped32 = distinct uint32 - UIntWrapped64 = distinct uint64 - LUIntWrapped32 = distinct uint32 - LUIntWrapped64 = distinct uint64 - - #Types which share an encoding. - PIntWrapped = PIntWrapped32 or PIntWrapped64 or enum - SIntWrapped = SIntWrapped32 or SIntWrapped64 - UIntWrapped = UIntWrapped32 or UIntWrapped64 or - byte or char or bool - LUIntWrapped* = LUIntWrapped32 or LUIntWrapped64 - - #Any wrapped VarInt types. - VarIntWrapped* = PIntWrapped or SIntWrapped or - UIntWrapped or LUIntWrapped - - #Every signed integer Type. - SIntegerTypes = PureSIntegerTypes or - PIntWrapped or SIntWrapped - - #Every unsigned integer Type. - UIntegerTypes = PureUIntegerTypes or UIntWrapped or LUIntWrapped - - #Every type valid for the VarInt wire type. - VarIntTypes* = SIntegerTypes or UIntegerTypes - -generateWrapper( - PInt, UIntegerTypes or SIntegerTypes, VarIntWrapped, - UIntegerTypes, UIntWrapped64, UIntWrapped32, - SIntegerTypes, PIntWrapped64, PIntWrapped32, - "LInt should only be used with integers (signed or unsigned)." -) - -generateWrapper( - SInt, SIntegerTypes, VarIntWrapped, - void, void, void, - SIntegerTypes, SIntWrapped64, SIntWrapped32, - "SInt should only be used with signed integers." -) - -generateWrapper( - LInt, UIntegerTypes, VarIntWrapped, - UIntegerTypes, LUIntWrapped64, LUIntWrapped32, - void, void, void, - "LInt should only be used with unsigned integers." -) - -#Used to specify how to encode/decode fields in an object. -template pint*() {.pragma.} -template sint*() {.pragma.} - -template unwrap*(value: VarIntWrapped): untyped = - when value is (PIntWrapped32 or SIntWrapped32): - int32(value) - elif value is (PIntWrapped64 or SIntWrapped64): - int64(value) - elif value is (UIntWrapped32 or LUIntWrapped32): - uint32(value) - elif value is (UIntWrapped64 or LUIntWrapped64): - uint64(value) - elif value is UIntWrapped: - value - elif value is enum: - int(value) - else: - {.fatal: "Tried to get the unwrapped value of a non-wrapped type. This should never happen.".} - -func encodeBinaryValue(value: VarIntWrapped): auto = - when sizeof(value) == 8: - result = cast[uint64](value) - else: - result = cast[uint32](value) - - mixin unwrap - when value is PIntWrapped: - if value.unwrap() < 0: - result = not result - elif value is SIntWrapped: - #This line is the formula exactly as described in the Protobuf docs. - #That said, it's quite verbose. - #The below formula which is actually used is much simpler and possibly faster. - #This is preserved to note it, but not to be used. - #result = (result shl 1) xor cast[type(result)](ashr(value.unwrap(), (sizeof(result) * 8) - 1)) - result = result shl 1 - if value.unwrap() < 0: - result = not result - elif value is UIntWrapped: - discard - else: - {.fatal: "Tried to get the binary value of an unrecognized VarInt type.".} - -func viSizeof(base: VarIntWrapped, raw: uint32 or uint64): int = - when base is PIntWrapped: - if base.unwrap() < 0: - return 10 - result = max((log2trunc(raw) + 7) div 7, 1) - -func encodeVarInt*( - res: var openArray[byte], - outLen: var int, - value: VarIntWrapped -): VarIntStatus = - #Verify the value fits into the specified encoding. - when value is LUIntWrapped: - when sizeof(value) == 8: - if value.unwrap() shr 63 != 0: - return VarIntStatus.Overflow - - #Get the binary value of whatever we're decoding. - #Beyond the above check, LibP2P uses the standard UInt encoding. - #That's why we perform this cast. - var raw = encodeBinaryValue(PInt(value.unwrap())) - else: - var raw = encodeBinaryValue(value) - - outLen = viSizeof(value, raw) - - #Verify there's enough bytes to store this value. - if res.len < outLen: - return VarIntStatus.Incomplete - - #Write the VarInt. - var i = 0 - while raw > type(raw)(VAR_INT_VALUE_MASK): - res[i] = byte(raw and type(raw)(VAR_INT_VALUE_MASK)) or VAR_INT_CONTINUATION_MASK - inc(i) - raw = raw shr 7 - - #If this was a positive number (PInt or UInt), or zig-zagged, we only need to write this last byte. - when value is PIntWrapped: - if value.unwrap() < 0: - #[ - To signify this is negative, this should be artifically padded to 10 bytes. - That said, we have to write the final pending byte left in raw, as well as masks until then. - #This iterates up to 9. - We don't immediately write the final pending byte and then loop. - Why? Because if all 9 bytes were used, it'll set the continuation flag when it shouldn't. - If all 9 bytes were used, the last byte is 0 anyways. - By setting raw to 0, which is pointless after the first loop, we avoid two conditionals. - ]# - while i < 9: - res[i] = VAR_INT_CONTINUATION_MASK or byte(raw) - inc(i) - raw = 0 - else: - res[i] = byte(raw) - else: - res[i] = byte(raw) - -func encodeVarInt*(value: VarIntWrapped): seq[byte] = - result = newSeq[byte](10) - var outLen: int - if encodeVarInt(result, outLen, value) != VarIntStatus.Success: - when value is LUIntWrapped: - {.fatal: "LibP2P VarInts require using the following signature: `encodeVarInt(var openArray[byte], outLen: var int, value: VarIntWrapped): VarIntStatus`.".} - else: - doAssert(false) - result.setLen(outLen) - -proc encodeVarInt*(stream: OutputStream, value: VarIntWrapped) {.inline.} = - stream.write(encodeVarInt(value)) - -func decodeBinaryValue[E]( - res: var E, - value: uint32 or uint64, - len: int -): VarIntStatus = - when (sizeof(E) != sizeof(value)) and (sizeof(E) != 1): - {.fatal: "Tried to decode a raw binary value into an encoding with a different size. This should never happen.".} - - when E is LUIntWrapped: - if res.unwrap() shr ((sizeof(res) * 8) - 1) == 1: - return VarIntStatus.Overflow - res = E(value) - - elif E is PIntWrapped: - if len == 10: - when E is enum: - type S = int - else: - type S = type(res.unwrap()) - res = E((-S(value)) - 1) - else: - res = E(value) - - elif E is SIntWrapped: - type S = type(res.unwrap()) - res = E(S(value shr 1) xor -S(value and 0b0000_0001)) - - elif E is UIntWrapped: - res = E(value) - - else: - {.fatal: "Tried to decode a raw binary value into an unrecognized type. This should never happen.".} - - return VarIntStatus.Success - -func decodeVarInt*( - bytes: openArray[byte], - inLen: var int, - res: var VarIntWrapped -): VarIntStatus = - when sizeof(res) == 8: - type U = uint64 - var maxBits = 64 - else: - type U = uint32 - var maxBits = 32 - - when (res is LUIntWrapped) and (sizeof(res) == 8): - maxBits = 63 - - var - value: U - offset = 0'i8 - next = VAR_INT_CONTINUATION_MASK - while (next and VAR_INT_CONTINUATION_MASK) != 0: - if inLen == bytes.len: - return VarIntStatus.Incomplete - next = bytes[inLen] - if (next and VAR_INT_VALUE_MASK) == 0: - inLen += 1 - offset += 7 - continue - - if (offset + log2trunc(next and VAR_INT_VALUE_MASK) + 1) > maxBits: - return VarIntStatus.Overflow - - value += (next and U(VAR_INT_VALUE_MASK)) shl offset - inLen += 1 - offset += 7 - - return decodeBinaryValue(res, value, inLen) - -proc decodeVarInt*[R, E]( - stream: InputStream, - returnType: typedesc[R], - encoding: typedesc[E] -): R = - var - bytes: seq[byte] - next: byte = VAR_INT_CONTINUATION_MASK - value: E - inLen: int - - while (next and VAR_INT_CONTINUATION_MASK) != 0: - if not stream.readable(): - raise newException(ProtobufEOFError, "Stream ended before the VarInt was finished.") - next = stream.read() - bytes.add(next) - - if decodeVarInt(bytes, inLen, value) != VarIntStatus.Success: - raise newException(ProtobufMessageError, "Attempted to decode an invalid VarInt.") - doAssert inLen == bytes.len - - #Removes a warning. - when value is R: - result = value - else: - result = R(value) diff --git a/protobuf_serialization/pb_option.nim b/protobuf_serialization/pb_option.nim deleted file mode 100644 index 661d7d8..0000000 --- a/protobuf_serialization/pb_option.nim +++ /dev/null @@ -1,37 +0,0 @@ -{.warning[UnusedImport]: off} -import sets - -type PBOption*[defaultValue: static[auto]] = object - some: bool - value: typeof(defaultValue) - -proc isNone*(opt: PBOption): bool {.inline.} = - not opt.some - -proc isSome*(opt: PBOption): bool {.inline.} = - opt.some - -proc get*(opt: PBOption): auto = - when opt.defaultValue is (object or seq or set or HashSet): - {.fatal: "PBOption can not be used with objects or repeated types."} - - if opt.some: - result = opt.value - else: - result = opt.defaultValue - -proc pbSome*[T](optType: typedesc[T], value: auto): T {.inline.} = - when value is (object or seq or set or HashSet): - {.fatal: "PBOption can not be used with objects or repeated types."} - - T( - some: true, - value: value - ) - -proc init*(opt: var PBOption, val: auto) = - opt.some = true - opt.value = val - -converter toValue*(opt: PBOption): auto {.inline.} = - opt.get() diff --git a/protobuf_serialization/proto_parser.nim b/protobuf_serialization/proto_parser.nim new file mode 100644 index 0000000..c4aa4a5 --- /dev/null +++ b/protobuf_serialization/proto_parser.nim @@ -0,0 +1,2 @@ +import protobuf_serialization/files/type_generator +export protoToTypes, import_proto3 diff --git a/protobuf_serialization/reader.nim b/protobuf_serialization/reader.nim index 04584f4..2a8e748 100644 --- a/protobuf_serialization/reader.nim +++ b/protobuf_serialization/reader.nim @@ -1,276 +1,157 @@ #Parses the Protobuf binary wire protocol into the specified type. -import options +import + std/[typetraits, sets], + stew/assign2, + stew/shims/macros, + faststreams/inputs, + serialization, + "."/[codec, internal, types] -import stew/shims/macros -import faststreams/inputs -import serialization +export inputs, serialization, codec, types -import internal -import types +proc readValueInternal[T: object](stream: InputStream, value: var T, silent: bool = false) -proc readVarInt[B, E]( +macro unsupported(T: typed): untyped = + error "Assignment of the type " & humaneTypeName(T) & " is not supported" + +template requireKind(header: FieldHeader, expected: WireKind) = + mixin number + if header.kind() != expected: + raise (ref ValueError)( + msg: "Unexpected data kind " & $(header.number()) & ": " & $header.kind() & + ", exprected " & $expected) + +proc readFieldInto[T: object]( stream: InputStream, - fieldVar: var B, - encoding: E, - key: ProtobufKey -) = - when E is not VarIntWrapped: - {.fatal: "Tried to read a VarInt without a specified encoding. This should never happen.".} - - if key.wire != VarInt: - raise newException(ProtobufMessageError, "Invalid wire type for a VarInt.") - - #Box the result back up. - box(fieldVar, stream.decodeVarInt(flatType(B), type(E))) - -proc readFixed[B](stream: InputStream, fieldVar: var B, key: ProtobufKey) = - type T = flatType(B) - var value: T - - when sizeof(T) == 8: - if key.wire != Fixed64: - raise newException(ProtobufMessageError, "Invalid wire type for a Fixed64.") - else: - if key.wire != Fixed32: - raise newException(ProtobufMessageError, "Invalid wire type for a Fixed32.") - - stream.decodeFixed(value) - box(fieldVar, value) - -include stdlib_readers - -proc readValueInternal[T](stream: InputStream, ty: typedesc[T], silent: bool = false): T - -proc readLengthDelimited[R, B]( - stream: InputStream, - rootType: typedesc[R], - fieldName: static string, - fieldVar: var B, - key: ProtobufKey -) = - if key.wire != LengthDelimited: - raise newException(ProtobufMessageError, "Invalid wire type for a length delimited sequence/object.") - - var - #We need to specify a bit quantity for decode to be satisfied. - #int64 won't work on int32 systems, as this eventually needs to be casted to int. - #We could just use the proper int size for the system. - #That said, a 2 GB buffer limit isn't a horrible idea from a security perspective. - #If anyone has a valid reason for one, let me know. - - #Uses PInt to ensure 31-bits are used, not 32-bits. - len = stream.decodeVarInt(int, PInt(int32)) - preResult: flatType(B) - if len < 0: - raise newException(ProtobufMessageError, "Length delimited buffer contained more than 2 GB of data.") - - when preResult is CastableLengthDelimitedTypes: - var byteResult: seq[byte] = newSeq[byte](len) - if not stream.readInto(byteResult): - raise newException(ProtobufEOFError, "Couldn't read the length delimited buffer from this stream.") - preResult = cast[type(preResult)](byteResult) - - else: - if not stream.readable(len): - raise newException(ProtobufEOFError, "Couldn't read the length delimited buffer from this stream despite expecting one.") - - stream.withReadableRange(len, substream): - when preResult is not LengthDelimitedTypes: - {.fatal: "Tried to read a Length Delimited value which we didn't recognize. This should never happen.".} - elif B.isStdlib(): - substream.stdlibFromProtobuf(rootType, fieldName, preResult) - elif preResult is (object or tuple): - preResult = substream.readValueInternal(type(preResult)) - else: - {.fatal: "Tried to read a Length Delimited type which wasn't actually Length Delimited. This should never happen.".} - - box(fieldVar, preResult) - -proc setField[T]( value: var T, - stream: InputStream, - key: ProtobufKey, - requiredSets: var HashSet[int] + header: FieldHeader, + ProtoType: type ) = - when T is (ref or ptr or Option): - {.fatal: "Ref or Ptr or Option made it to setField. This should never happen.".} + header.requireKind(WireKind.LengthDelim) - elif T is (seq or set or HashSet): - template merge[I]( - stdlib: var (seq[I] or set[I] or HashSet[I]), - value: I - ) = - when stdlib is seq: - stdlib.add(value) - else: - stdlib.incl(value) + let len = stream.readLength() + if len > 0: + # TODO: https://github.com/status-im/nim-faststreams/issues/31 + # TODO: check that all bytes were read + # stream.withReadableRange(len, inner): + # inner.readValueInternal(value) - type U = value.getUnderlyingType() - #Unpacked seq of numbers. - if key.wire != LengthDelimited: - var next: U - when flatType(U) is VarIntWrapped: - stream.readVarInt(next, next, key) - elif flatType(U) is FixedWrapped: - stream.readFixed(next, key) - else: - if true: - raise newException(ProtobufMessageError, "Reading into an unpacked seq yet value is a number.") - merge(value, next) - #Packed seq of numbers/unpacked seq of objects. - else: - when flatType(U) is (VarIntWrapped or FixedWrapped): - var newValues: seq[U] - stream.readLengthDelimited(type(value), "", newValues, key) - for newValue in newValues: - merge(value, newValue) - else: - var - next: flatType(U) - boxed: U - stream.readLengthDelimited(U, "", next, key) - box(boxed, next) - merge(value, boxed) - - elif T is not (object or tuple): - when T is VarIntWrapped: - stream.readVarInt(value, value, key) - elif T is FixedWrapped: - stream.readFixed(value, key) - elif T is (PlatformDependentTypes or VarIntTypes or FixedTypes): - {.fatal: "Reading into a number requires specifying both the amount of bits via the type, as well as the encoding format.".} - else: - stream.readLengthDelimited(type(value), "", value, key) + var tmp = newSeqUninitialized[byte](len) + if not stream.readInto(tmp): + raise (ref ValueError)(msg: "not enough bytes") + memoryInput(tmp).readValueInternal(value) +proc readFieldInto[T: not object and (seq[byte] or not seq)]( + stream: InputStream, + value: var T, + header: FieldHeader, + ProtoType: type +) = + when ProtoType is SomeVarint: + header.requireKind(WireKind.Varint) + assign(value, T(stream.readValue(ProtoType))) + elif ProtoType is SomeFixed64: + header.requireKind(WireKind.Fixed64) + assign(value, T(stream.readValue(ProtoType))) + elif ProtoType is SomeLengthDelim: + header.requireKind(WireKind.LengthDelim) + assign(value, T(stream.readValue(ProtoType))) + elif ProtoType is SomeFixed32: + header.requireKind(WireKind.Fixed32) + assign(value, T(stream.readValue(ProtoType))) else: - #This iterative approach is extemely poor. - #See https://github.com/kayabaNerve/nim-protobuf-serialization/issues/8. - var - keyNumber: int = key.number - found: bool = false - when T.hasCustomPragma(protobuf2): - var rSI: int = -1 + static: unsupported(ProtoType) - enumInstanceSerializedFields(value, fieldName, fieldVar): - discard fieldName - when T.hasCustomPragma(protobuf2): - inc(rSI) +proc readFieldInto[T: not byte]( + stream: InputStream, + value: var seq[T], + header: FieldHeader, + ProtoType: type +) = + value.add(default(T)) + stream.readFieldInto(value[^1], header, ProtoType) - when fieldVar is PlatformDependentTypes: - {.fatal: "Reading into a number requires specifying the amount of bits via the type.".} +proc readFieldInto( + stream: InputStream, + value: var PBOption, + header: FieldHeader, + ProtoType: type +) = + stream.readFieldInto(value.mget(), header, ProtoType) - if keyNumber == fieldVar.getCustomPragmaVal(fieldNumber): - found = true - when T.hasCustomPragma(protobuf2): - requiredSets.excl(rSI) +proc readFieldPackedInto[T]( + stream: InputStream, + value: var seq[T], + header: FieldHeader, + ProtoType: type +) = + # TODO make more efficient + var + bytes = seq[byte](stream.readValue(pbytes)) + inner = memoryInput(bytes) + while inner.readable(): + value.add(default(T)) - var - blank: flatType(fieldVar) - flattened = flatMap(fieldVar).get(blank) - when blank is (seq or set or HashSet): - type U = flattened.getUnderlyingType() - when U is (VarIntWrapped or FixedWrapped): - var castedVar = flattened - elif U is (VarIntTypes or FixedTypes): - when T.hasCustomPragmaFixed(fieldName, pint): - #Nim encounters an error when doing `type C = PInt(U)`. - var - pointless: U - C = PInt(pointless) - elif T.hasCustomPragmaFixed(fieldName, sint): - var - pointless: U - C = SInt(pointless) - elif T.hasCustomPragmaFixed(fieldName, fixed): - var - pointless: U - C = Fixed(pointless) + let kind = when ProtoType is SomeVarint: + WireKind.Varint + elif ProtoType is SomeFixed32: + WireKind.Fixed32 + else: + static: doAssert ProtoType is SomeFixed64 + ProtoType.SomeFixed64 - when flattened is seq: - var castedVar = cast[seq[type(C)]](flattened) - elif flattened is set: - var castedVar = cast[set[type(C)]](flattened) - elif flattened is HashSet: - var castedVar = cast[HashSet[type(C)]](flattened) - else: - var castedVar = flattened - var requiredSets: HashSet[int] = initHashSet[int]() - castedVar.setField(stream, key, requiredSets) + inner.readFieldInto(value[^1], FieldHeader.init(header.number, kind), ProtoType) - flattened = cast[type(flattened)](castedVar) - else: - #Only calculate the encoding for VarInt. - #In every other case, the type is enough. - when flattened is VarIntWrapped: - stream.readVarInt(flattened, flattened, key) +proc readValueInternal[T: object](stream: InputStream, value: var T, silent: bool = false) = + const + isProto2: bool = T.isProto2() - elif flattened is FixedWrapped: - stream.readFixed(flattened, key) - - elif flattened is VarIntTypes: - when T.hasCustomPragmaFixed(fieldName, pint): - stream.readVarInt(flattened, PInt(flattened), key) - elif T.hasCustomPragmaFixed(fieldName, sint): - stream.readVarInt(flattened, SInt(flattened), key) - elif T.hasCustomPragmaFixed(fieldName, fixed): - stream.readFixed(flattened, key) - else: - {.fatal: "Encoding pragma specified yet no enoding matched. This should never happen.".} - - else: - stream.readLengthDelimited(type(value), fieldName, flattened, key) - - box(fieldVar, flattened) - break - - if not found: - raise newException(ProtobufMessageError, "Message encoded an unknown field number.") - -proc readValueInternal[T](stream: InputStream, ty: typedesc[T], silent: bool = false): T = - static: verifySerializable(flatType(T)) - - var requiredSets: HashSet[int] = initHashSet[int]() - when (flatType(result) is object) and (not flatType(result).isStdlib()): - when ty.hasCustomPragma(protobuf2): + when isProto2: + var requiredSets: HashSet[int] + if not silent: var i: int = -1 - enumInstanceSerializedFields(result, fieldName, fieldVar): + enumInstanceSerializedFields(value, fieldName, fieldVar): inc(i) - when ty.hasCustomPragmaFixed(fieldName, required): + + when T.hasCustomPragmaFixed(fieldName, required): requiredSets.incl(i) while stream.readable(): - result.setField(stream, stream.readProtobufKey(), requiredSets) + let header = stream.readHeader() + var i = -1 + enumInstanceSerializedFields(value, fieldName, fieldVar): + inc i + const + fieldNum = T.fieldNumberOf(fieldName) - if (requiredSets.len != 0) and (not silent): - raise newException(ProtobufReadError, "Message didn't encode a required field.") + if header.number() == fieldNum: + when isProto2: + if not silent: requiredSets.excl i -proc readValue*(reader: ProtobufReader, value: var auto) = - when value is Option: - {.fatal: "Can't decode directly into an Option.".} + protoType(ProtoType, T, typeof(fieldVar), fieldName) - when (flatType(value) is object) and (not flatType(value).isStdlib()): - static: - for c in $type(value): - if c == ' ': - raise newException(Exception, "Told to read into an inlined type; every type read into must have a proper type definition: " & $type(value)) - when type(value).hasCustomPragma(protobuf2): - if not reader.stream.readable(): - enumInstanceSerializedFields(value, fieldName, fieldVar): - when type(value).hasCustomPragmaFixed(fieldName, required): - raise newException(ProtobufReadError, "Message didn't encode a required field.") + # TODO should we allow reading packed fields into non-repeated fields? + when ProtoType is SomePrimitive and fieldVar is seq and fieldVar isnot seq[byte]: + if header.kind() == WireKind.LengthDelim: + stream.readFieldPackedInto(fieldVar, header, ProtoType) + else: + stream.readFieldInto(fieldVar, header, ProtoType) + else: + stream.readFieldInto(fieldVar, header, ProtoType) + + when isProto2: + if (requiredSets.len != 0): + raise newException( + ProtobufReadError, + "Message didn't encode a required field: " & $requiredSets) + +proc readValue*[T: object](reader: ProtobufReader, value: var T) = + static: verifySerializable(T) + + # TODO skip length header try: - if reader.stream.readable(): - if reader.keyOverride.isNone(): - box(value, reader.stream.readValueInternal(flatType(type(value)))) - else: - var preResult: flatType(type(value)) - while reader.stream.readable(): - var requiredSets: HashSet[int] = initHashSet[int]() - preResult.setField(reader.stream, reader.keyOverride.get(), requiredSets) - box(value, preResult) - except ProtobufReadError as e: - raise e + reader.stream.readValueInternal(value) finally: if reader.closeAfter: reader.stream.close() diff --git a/protobuf_serialization/stdlib_readers.nim b/protobuf_serialization/stdlib_readers.nim deleted file mode 100644 index 919b917..0000000 --- a/protobuf_serialization/stdlib_readers.nim +++ /dev/null @@ -1,116 +0,0 @@ -#Included by reader. - -import sets - -import stew/shims/macros -import faststreams - -import internal -import types - -proc decodeNumber[T, E]( - stream: InputStream, - next: var T, - encoding: typedesc[E] -) = - var flattened: flatType(T) - when E is VarIntWrapped: - flattened = stream.decodeVarInt(type(flattened), encoding) - elif E is FixedWrapped: - stream.decodeFixed(flattened) - else: - {.fatal: "Trying to decode a number which isn't wrapped. This should never happen.".} - box(next, flattened) - -proc readValue*(reader: ProtobufReader, value: var auto) - -proc stdlibFromProtobuf[R]( - stream: InputStream, - _: typedesc[R], - unusedFieldName: static string, - value: var string -) = - value = newString(stream.totalUnconsumedBytes) - for c in 0 ..< value.len: - value[c] = char(stream.read()) - -proc stdlibFromProtobuf[R]( - stream: InputStream, - _: typedesc[R], - unusedFieldName: static string, - value: var cstring -) = - var preValue = newString(stream.totalUnconsumedBytes) - for c in 0 ..< preValue.len: - preValue[c] = char(stream.read()) - value = preValue - -proc stdlibFromProtobuf[R, T]( - stream: InputStream, - ty: typedesc[R], - fieldName: static string, - seqInstance: var seq[T] -) = - type fType = flatType(T) - var blank: T - while stream.readable(): - seqInstance.add(blank) - - when fType is (VarIntWrapped or FixedWrapped): - stream.decodeNumber(seqInstance[^1], type(seqInstance[^1])) - - elif fType is VarIntTypes: - when fieldName is "": - {.fatal: "A standard lib type didn't specify the encoding to use for a number.".} - - when R.hasCustomPragmaFixed(fieldName, pint): - stream.decodeNumber(seqInstance[^1], PInt(type(seqInstance[^1]))) - elif R.hasCustomPragmaFixed(fieldName, sint): - stream.decodeNumber(seqInstance[^1], SInt(type(seqInstance[^1]))) - elif R.hasCustomPragmaFixed(fieldName, fixed): - stream.decodeNumber(seqInstance[^1], Fixed(type(seqInstance[^1]))) - - elif fType is FixedTypes: - when fieldName is "": - {.fatal: "A standard lib type didn't specify the encoding to use for a number.".} - - stream.decodeNumber(seqInstance[^1], Fixed(type(seqInstance[^1]))) - - elif fType is (cstring or string): - var len = stream.decodeVarInt(int, PInt(int32)) - if len < 0: - raise newException(ProtobufMessageError, "String longer than 2 GB specified.") - - if not stream.readable(len): - raise newException(ProtobufEOFError, "Length delimited buffer is bigger than the rest of the stream.") - stream.withReadableRange(len, substream): - substream.stdlibFromProtobuf(ty, fieldName, seqInstance[^1]) - - elif fType is CastableLengthDelimitedTypes: - ProtobufReader.init(substream, some(T.wireType), false).readValue(seqInstance[^1]) - - elif (fType is object) or fType.isStdlib(): - let len = stream.decodeVarInt(int, PInt(int32)) - if len < 0: - raise newException(ProtobufMessageError, "Length delimited buffer contained more than 2 GB of data.") - elif len == 0: - continue - elif not stream.readable(len): - raise newException(ProtobufEOFError, "Length delimited buffer doesn't have enough data to read the next object.") - - stream.withReadableRange(len, substream): - ProtobufReader.init(substream, closeAfter = false).readValue(seqInstance[^1]) - - else: - {.fatal: "Tried to decode an unrecognized object used in a stdlib type.".} - -proc stdlibFromProtobuf[R, T]( - stream: InputStream, - ty: typedesc[R], - fieldName: static string, - setInstance: var (set[T] or HashSet[T]) -) = - var seqInstance: seq[T] - stream.stdlibFromProtobuf(ty, fieldName, seqInstance) - for value in seqInstance: - setInstance.incl(value) diff --git a/protobuf_serialization/stdlib_writers.nim b/protobuf_serialization/stdlib_writers.nim deleted file mode 100644 index a0d3dc1..0000000 --- a/protobuf_serialization/stdlib_writers.nim +++ /dev/null @@ -1,136 +0,0 @@ -#Included by writer. - -import sets -import sequtils - -import stew/shims/macros - -import internal -import types - -proc encodeNumber[T](stream: OutputStream, value: T) = - when value is VarIntWrapped: - stream.encodeVarInt(value) - elif value is FixedWrapped: - stream.encodeFixed(value) - else: - {.fatal: "Trying to encode a number which isn't wrapped. This should never happen.".} - -proc stdlibToProtobuf[R]( - stream: OutputStream, - _: typedesc[R], - unusedFieldName: static string, - fieldNumber: int, - value: cstring or string -) = - stream.write(cast[seq[byte]]($value)) - -proc stdlibToProtobuf[R, T]( - stream: OutputStream, - ty: typedesc[R], - fieldName: static string, - fieldNumber: int, - arrInstance: openArray[T] -) = - #Get the field number and create a key. - var key: seq[byte] - - type fType = flatType(T) - when fType is FixedTypes: - var hasFixed = false - when (R is (object or tuple)) and (not R.isStdlib()): - hasFixed = R.hasCustomPragmaFixed(fieldName, fixed) - - when fType is (VarIntTypes or FixedTypes): - when fType is FixedTypes: - if hasFixed: - key = newProtobufKey( - fieldNumber, - when sizeof(fType) == 8: - Fixed64 - else: - Fixed32 - ) - else: - key = newProtobufKey(fieldNumber, VarInt) - else: - key = newProtobufKey(fieldNumber, VarInt) - else: - key = newProtobufKey(fieldNumber, LengthDelimited) - - const singleBuffer = type(arrInstance).singleBufferable() - for value in arrInstance: - if not singleBuffer: - stream.write(key) - - when fType is (VarIntWrapped or FixedWrapped): - let possibleNumber = flatMap(value) - var blank: fType - stream.encodeNumber(possibleNumber.get(blank)) - - elif fType is VarIntTypes: - when fieldName is "": - {.fatal: "A standard lib type didn't specify the encoding to use for a number.".} - - let possibleNumber = flatMap(value) - var blank: fType - - when R.hasCustomPragmaFixed(fieldName, pint): - stream.encodeNumber(PInt(possibleNumber.get(blank))) - elif R.hasCustomPragmaFixed(fieldName, sint): - stream.encodeNumber(SInt(possibleNumber.get(blank))) - elif R.hasCustomPragmaFixed(fieldName, fixed): - stream.encodeNumber(Fixed(possibleNumber.get(blank))) - - elif fType is FixedTypes: - when fieldName is "": - {.fatal: "A standard lib type didn't specify the encoding to use for a number.".} - - let possibleNumber = flatMap(value) - var blank: fTypeflatType(T) - stream.encodeNumber(Fixed(possibleNumber.get(blank))) - - elif fType is (cstring or string): - var cursor = stream.delayVarSizeWrite(5) - let startPos = stream.pos - stream.stdlibToProtobuf(ty, fieldName, fieldNumber, flatMap(value).get("")) - cursor.finalWrite(encodeVarInt(PInt(int32(stream.pos - startPos)))) - - elif fType is CastableLengthDelimitedTypes: - let toEncode = flatMap(value).get(T(@[])) - if toEncode.len == 0: - return - stream.write(encodeVarInt(PInt(toEncode.len))) - stream.write(cast[seq[byte]](toEncode)) - - elif (fType is (object or tuple)) or fType.isStdlib(): - var cursor = stream.delayVarSizeWrite(5) - let startPos = stream.pos - - stream.writeValueInternal(value) - - cursor.finalWrite(encodeVarInt(PInt(int32(stream.pos - startPos)))) - - else: - {.fatal: "Tried to encode an unrecognized object used in a stdlib type.".} - -proc stdlibToProtobuf[R, T]( - stream: OutputStream, - ty: typedesc[R], - fieldName: static string, - fieldNumber: int, - setInstance: set[T] -) = - var seqInstance: seq[T] - for value in setInstance: - seqInstance.add(value) - stream.stdlibToProtobuf(ty, fieldName, fieldNumber, seqInstance) - -proc stdlibToProtobuf[R, T]( - stream: OutputStream, - ty: typedesc[R], - fieldName: static string, - fieldNumber: int, - setInstance: HashSet[T] -) {.inline.} = - stream.stdlibToProtobuf(ty, fieldName, fieldNumber, setInstance.toSeq()) diff --git a/protobuf_serialization/types.nim b/protobuf_serialization/types.nim index 9fe54c9..5729b3f 100644 --- a/protobuf_serialization/types.nim +++ b/protobuf_serialization/types.nim @@ -1,24 +1,20 @@ #Types/common data exported for use outside of this library. -import faststreams -import serialization/errors +import + faststreams, + serialization/errors -import numbers/varint -import numbers/fixed -export varint, fixed - -import internal -export protobuf2, protobuf3, fieldNumber, required -export ProtobufError, ProtobufReadError, ProtobufEOFError, ProtobufMessageError - -import pb_option -export pb_option +export faststreams, errors type - ProtobufFlags* = enum - VarIntLengthPrefix, - UIntLELengthPrefix, - UIntBELengthPrefix + ProtobufError* = object of SerializationError + + ProtobufReadError* = object of ProtobufError + ProtobufEOFError* = object of ProtobufReadError + ProtobufMessageError* = object of ProtobufReadError + + ProtobufFlags* = uint8 # enum + # VarIntLengthPrefix, # TODO needs fixing ProtobufWriter* = object stream*: OutputStream @@ -26,9 +22,24 @@ type ProtobufReader* = ref object stream*: InputStream - keyOverride*: Option[ProtobufKey] closeAfter*: bool + PBOption*[defaultValue: static[auto]] = object + some: bool + value: typeof(defaultValue) + +# Message type annotations +template proto2*() {.pragma.} +template proto3*() {.pragma.} + +# Field annotations +template fieldNumber*(num: int) {.pragma.} +template required*() {.pragma.} +template packed*(v: bool) {.pragma.} +template pint*() {.pragma.} # encode as `intXX` +template sint*() {.pragma.} # encode as `sintXX` +template fixed*() {.pragma.} # encode as `fixedXX` + func init*( T: type ProtobufWriter, stream: OutputStream, @@ -39,10 +50,10 @@ func init*( func init*( T: type ProtobufReader, stream: InputStream, - key: Option[ProtobufKey] = none(ProtobufKey), + # key: Option[ProtobufKey] = none(ProtobufKey), closeAfter: bool = true ): T {.inline.} = - T(stream: stream, keyOverride: key, closeAfter: closeAfter) + T(stream: stream, closeAfter: closeAfter) #This was originally called buffer, and retuned just the output. #That said, getting the output purges the stream, and doesn't close it. @@ -51,3 +62,32 @@ func init*( proc finish*(writer: ProtobufWriter): seq[byte] = result = writer.stream.getOutput() writer.stream.close() + +func isNone*(opt: PBOption): bool {.inline.} = + not opt.some + +func isSome*(opt: PBOption): bool {.inline.} = + opt.some + +func get*(opt: PBOption): auto = + if opt.some: + opt.value + else: + opt.defaultValue + +template mget*(opt: var PBOption): untyped = + opt.some = true + opt.value + +func pbSome*[T: PBOption](optType: typedesc[T], value: auto): T {.inline.} = + T( + some: true, + value: value + ) + +func init*(opt: var PBOption, val: auto) = + opt.some = true + opt.value = val + +converter toValue*(opt: PBOption): auto {.inline.} = + opt.get() diff --git a/protobuf_serialization/writer.nim b/protobuf_serialization/writer.nim index 841726c..0d95146 100644 --- a/protobuf_serialization/writer.nim +++ b/protobuf_serialization/writer.nim @@ -1,236 +1,139 @@ #Writes the specified type into a buffer using the Protobuf binary wire format. -import options +import + std/typetraits, + stew/shims/macros, + faststreams/outputs, + serialization, + "."/[codec, internal, types] -import stew/shims/macros -import faststreams/outputs -import serialization +export outputs, serialization, codec, types -import internal -import types +proc writeValue*[T: object](stream: OutputStream, value: T) -proc writeVarInt( - stream: OutputStream, - fieldNum: int, - value: VarIntWrapped, - omittable: static bool -) = - let bytes = encodeVarInt(value) - when omittable: - if (bytes.len == 1) and (bytes[0] == 0): - return - stream.writeProtobufKey(fieldNum, VarInt) - stream.write(bytes) +proc writeField( + stream: OutputStream, fieldNum: int, fieldVal: auto, ProtoType: type UnsupportedType) = + # TODO turn this into an extension point + unsupportedProtoType ProtoType.FieldType, ProtoType.RootType, ProtoType.fieldName -proc writeFixed( - stream: OutputStream, - fieldNum: int, - value: auto, - omittable: static bool -) = - when sizeof(value) == 8: - let wire = Fixed64 - else: - let wire = Fixed32 - when omittable: - if value.unwrap() == 0: - return +proc writeField*[T: object](stream: OutputStream, fieldNum: int, fieldVal: T) = + # TODO Pre-compute size of inner object then write it without the intermediate + # memory output + var inner = memoryOutput() + inner.writeValue(fieldVal) + let bytes = inner.getOutput() + stream.writeField(fieldNum, pbytes(bytes)) - stream.writeProtobufKey(fieldNum, wire) - stream.encodeFixed(value) +proc writeField[T: object and not PBOption]( + stream: OutputStream, fieldNum: int, fieldVal: T, ProtoType: type) = + stream.writeField(fieldNum, fieldVal) -proc writeValueInternal[T](stream: OutputStream, value: T) +proc writeField[T: not object]( + stream: OutputStream, fieldNum: int, fieldVal: T, ProtoType: type) = + stream.writeField(fieldNum, ProtoType(fieldVal)) -#stdlib types toProtobuf's. inlined as it needs access to the writeValue function. -include stdlib_writers +proc writeField( + stream: OutputStream, fieldNum: int, fieldVal: PBOption, ProtoType: type) = + if fieldVal.isSome(): # TODO required field checking + stream.writeField(fieldNum, fieldVal.get(), ProtoType) -proc writeLengthDelimited[T]( - stream: OutputStream, - fieldNum: int, - rootType: typedesc[T], - fieldName: static string, - flatValue: LengthDelimitedTypes, - omittable: static bool -) = - const stdlib = type(flatValue).isStdlib() +proc writeFieldPacked*[T: not byte, ProtoType: SomePrimitive]( + output: OutputStream, field: int, values: openArray[T], _: type ProtoType) = + doAssert validFieldNumber(field) - var cursor = stream.delayVarSizeWrite(10) - let startPos = stream.pos + # Packed encoding uses a length-delimited field byte length of the sum of the + # byte lengths of each field followed by the header-free contents + output.write( + toBytes(FieldHeader.init(field, WireKind.LengthDelim))) - #Byte seqs. - when flatValue is CastableLengthDelimitedTypes: - if flatValue.len == 0: - cursor.finalWrite([]) - return - stream.write(cast[seq[byte]](flatValue)) - - #Standard lib types which use custom converters, instead of encoding the literal Nim representation. - elif stdlib: - stream.stdlibToProtobuf(rootType, fieldName, fieldNum, flatValue) - - #Nested object which even if the sub-value is empty, should be encoded as long as it exists. - elif rootType.isPotentiallyNull(): - writeValueInternal(stream, flatValue) - - #Object which should only be encoded if it has data. - elif flatValue is (object or tuple): - writeValueInternal(stream, flatValue) - - else: - {.fatal: "Tried to write a Length Delimited type which wasn't actually Length Delimited.".} - - const singleBuffer = type(flatValue).singleBufferable() - if ( - ( - #The underlying type of the standard library container is packable. - singleBuffer or ( - #This is a object, not a seq or something converted to a seq (stdlib type). - (not stdlib) and (flatValue is (object or tuple)) - ) - ) and ( - #The length changed, meaning this object is empty. - (stream.pos != startPos) or - #The object is empty, yet it exists, which is important as it can not exist. - rootType.isPotentiallyNull() - ) - ): - cursor.finalWrite(newProtobufKey(fieldNum, LengthDelimited) & encodeVarInt(PInt(int32(stream.pos - startPos)))) - else: - when omittable: - cursor.finalWrite([]) + const canCopyMem = + ProtoType is SomeFixed32 or ProtoType is SomeFixed64 or ProtoType is pbool + let dlength = + when canCopyMem: + values.len() * sizeof(T) else: - cursor.finalWrite(newProtobufKey(fieldNum, LengthDelimited) & encodeVarInt(PInt(int32(0)))) + var total = 0 + for item in values: + total += vsizeof(ProtoType(item)) + total + output.write(toBytes(puint64(dlength))) -proc writeFieldInternal[T, R]( - stream: OutputStream, - fieldNum: int, - value: T, - rootType: typedesc[R], - fieldName: static string -) = - when flatType(value) is SomeFloat: - when rootType.hasCustomPragmaFixed(fieldName, pfloat32): - static: verifySerializable(type(Float32(value))) - elif rootType.hasCustomPragmaFixed(fieldName, pfloat64): - static: verifySerializable(type(Float64(value))) - else: - {.fatal: "Float in object did not have an encoding pragma attached.".} + when canCopyMem: + if values.len > 0: + output.write( + cast[ptr UncheckedArray[byte]]( + unsafeAddr values[0]).toOpenArray(0, dlength - 1)) else: - static: verifySerializable(flatType(T)) + for value in values: + output.write(toBytes(ProtoType(value))) - let flattenedOption = value.flatMap() - if flattenedOption.isNone(): - return - let flattened = flattenedOption.get() +proc writeValue*[T: object](stream: OutputStream, value: T) = + const + isProto2: bool = T.isProto2() + isProto3: bool = T.isProto3() + static: doAssert isProto2 xor isProto3 - when (flatType(R) is not object) or (flatType(R).isStdlib()): - const omittable: bool = true - else: - when R is Option: - {.fatal: "Can't directly write an Option of an object.".} - const omittable: bool = ( - (fieldName == "") or - (flatType(T).isStdlib()) or - rootType.hasCustomPragma(protobuf3) - ) + enumInstanceSerializedFields(value, fieldName, fieldVal): + const + fieldNum = T.fieldNumberOf(fieldName) - when flattened is VarIntWrapped: - stream.writeVarInt(fieldNum, flattened, omittable) - elif flattened is FixedWrapped: - stream.writeFixed(fieldNum, flattened, omittable) - else: - stream.writeLengthDelimited(fieldNum, R, fieldName, flattened, omittable) + type + FlatType = flatType(fieldVal) -proc writeField*[T]( - writer: ProtobufWriter, - fieldNum: int, - value: T -) {.inline.} = - writer.stream.writeFieldInternal(fieldNum, value, type(value), "") + protoType(ProtoType, T, FlatType, fieldName) -proc writeValueInternal[T](stream: OutputStream, value: T) = - static: verifySerializable(flatType(T)) - - let flattenedOption = value.flatMap() - if flattenedOption.isNone(): - return - let flattened = flattenedOption.get() - - when type(flattened).isStdlib(): - stream.writeFieldInternal(1, flattened, type(value), "") - elif flattened is (object or tuple): - enumInstanceSerializedFields(flattened, fieldName, fieldVal): - discard fieldName - const fieldNum = getCustomPragmaVal(fieldVal, fieldNumber) - let flattenedFieldOption = fieldVal.flatMap() - if flattenedFieldOption.isSome(): - let flattenedField = flattenedFieldOption.get() - when flattenedField is ((not (VarIntWrapped or FixedWrapped)) and (VarIntTypes or FixedTypes)): - when flattenedField is VarIntTypes: - const - hasPInt = flatType(value).hasCustomPragmaFixed(fieldName, pint) - hasSInt = flatType(value).hasCustomPragmaFixed(fieldName, sint) - hasFixed = flatType(value).hasCustomPragmaFixed(fieldName, fixed) - when hasPInt: - stream.writeFieldInternal(fieldNum, PInt(flattenedField), type(value), fieldName) - elif hasSInt: - stream.writeFieldInternal(fieldNum, SInt(flattenedField), type(value), fieldName) - elif hasFixed: - stream.writeFieldInternal(fieldNum, Fixed(flattenedField), type(value), fieldName) - else: - {.fatal: "Encoding pragma specified yet no enoding matched. This should never happen.".} - - elif flattenedField is FixedTypes: - stream.writeFieldInternal(fieldNum, flattenedField, type(value), fieldName) - - else: - {.fatal: "Attempting to handle an unknown number type. This should never happen.".} - else: - when flattenedField is enum: - stream.writeFieldInternal(fieldNum, PInt(flattenedField), type(value), fieldName) - else: - stream.writeFieldInternal(fieldNum, flattenedField, type(value), fieldName) - else: - stream.writeFieldInternal(1, flattened, type(value), "") - -proc writeValue*[T](writer: ProtobufWriter, value: T) = - var - cursor: VarSizeWriteCursor - startPos: int - - if ( - writer.flags.contains(VarIntLengthPrefix) or - writer.flags.contains(UIntLELengthPrefix) or - writer.flags.contains(UIntBELengthPrefix) - ): - cursor = writer.stream.delayVarSizeWrite(5) - startPos = writer.stream.pos - - writer.stream.writeValueInternal(value) - - if ( - writer.flags.contains(VarIntLengthPrefix) or - writer.flags.contains(UIntLELengthPrefix) or - writer.flags.contains(UIntBELengthPrefix) - ): - var len = uint32(writer.stream.pos - startPos) - if len == 0: - cursor.finalWrite([]) - elif writer.flags.contains(VarIntLengthPrefix): - var viLen = encodeVarInt(PInt(len)) - if viLen.len == 0: - cursor.finalWrite([byte(0)]) + when FlatType is seq and FlatType isnot seq[byte]: + const + isPacked = T.isPacked(fieldName).get(isProto3) + when isPacked and ProtoType is SomePrimitive: + stream.writeFieldPacked(fieldNum, fieldVal, ProtoType) else: - cursor.finalWrite(viLen) - elif writer.flags.contains(UIntLELengthPrefix): - var temp: array[sizeof(len), byte] - for i in 0 ..< sizeof(len): - temp[i] = byte(len and LAST_BYTE) - len = len shr 8 - cursor.finalWrite(temp) - elif writer.flags.contains(UIntBELengthPrefix): - var temp: array[sizeof(len), byte] - for i in 0 ..< sizeof(len): - temp[i] = byte(len shr ((sizeof(len) - 1) * 8)) - len = len shl 8 - cursor.finalWrite(temp) + for i in 0.. encoded.len - check decodeVarInt(prefixed[0 ..< (prefixed.len - encoded.len)], inLen, res) == VarIntStatus.Success - check inLen == (prefixed.len - encoded.len) - check res.unwrap() == encoded.len - test "Can encode/decode a wrapper object": let obj = Wrapped( d: 300, e: 200, - f: Basic(a: 100, b: "Test string.", c: 'C'), + f: Basic(a: 100, b: "Test string."), # TODO, c: 'C'), g: "Other test string.", h: true ) @@ -161,17 +43,17 @@ suite "Test Object Encoding/Decoding": obj = Wrapped( d: 300, e: 200, - f: Basic(a: 100, b: "Test string.", c: 'C'), + f: Basic(a: 100, b: "Test string."), # c: 'C'), g: "Other test string.", h: true ) - writer = ProtobufWriter.init(memoryOutput()) + writer = memoryOutput() - writer.writeField(1, SInt(obj.d)) + writer.writeField(1, sint32(obj.d)) writer.writeField(3, obj.f) - writer.writeField(4, obj.g) + writer.writeField(4, pstring(obj.g)) - let result = Protobuf.decode(writer.finish(), type(Wrapped)) + let result = Protobuf.decode(writer.getOutput(), type(Wrapped)) check result.d == obj.d check result.f == obj.f check result.g == obj.g @@ -183,112 +65,27 @@ suite "Test Object Encoding/Decoding": obj = Wrapped( d: 300, e: 200, - f: Basic(a: 100, b: "Test string.", c: 'C'), + f: Basic(a: 100, b: "Test string."), # c: 'C'), g: "Other test string.", h: true ) - writer = ProtobufWriter.init(memoryOutput()) + writer = memoryOutput() writer.writeField(3, obj.f) - writer.writeField(1, SInt(obj.d)) - writer.writeField(2, SInt(obj.e)) - writer.writeField(5, obj.h) - writer.writeField(4, obj.g) + writer.writeField(1, sint64(obj.d)) + writer.writeField(2, sint64(obj.e)) + writer.writeField(5, pbool(obj.h)) + writer.writeField(4, pstring(obj.g)) - check Protobuf.decode(writer.finish(), type(Wrapped)) == obj + check Protobuf.decode(writer.getOutput(), type(Wrapped)) == obj test "Can read repeated fields": let - writer = ProtobufWriter.init(memoryOutput()) + writer = memoryOutput() basic: Basic = Basic(b: "Initial string.") repeated = "Repeated string." - writer.writeField(2, basic.b) - writer.writeField(2, repeated) + writer.writeField(2, pstring(basic.b)) + writer.writeField(2, pstring(repeated)) - check Protobuf.decode(writer.finish(), type(Basic)) == Basic(b: repeated) - - test "Can read nested objects": - let obj: Nested = Nested( - child: Nested( - data: "Child data." - ), - data: "Parent data." - ) - check Protobuf.decode(Protobuf.encode(obj), type(Nested)) == obj - - test "Can read pointered objects": - var ptrd = Pointered() - ptrd.x = cast[ptr int32](alloc0(sizeof(int32))) - ptrd.x[] = 5 - check Protobuf.decode(Protobuf.encode(ptrd), Pointered).x[] == ptrd.x[] - - var ptrPtrd: PtrPointered = addr ptrd - ptrPtrd.x = cast[ptr int32](alloc0(sizeof(int32))) - ptrPtrd.x[] = 8 - check Protobuf.decode(Protobuf.encode(ptrPtrd), PtrPointered).x[] == ptrPtrd.x[] - - test "Enum in object": - var x = TestObject(x: One) - check Protobuf.decode(Protobuf.encode(x), TestObject) == x - - var y = TestObject(x: Two) - check Protobuf.decode(Protobuf.encode(y), TestObject) == y - - var z = TestObject(x: NegOne) - check Protobuf.decode(Protobuf.encode(z), TestObject) == z - - var v = TestObject(x: NegTwo) - check Protobuf.decode(Protobuf.encode(v), TestObject) == v - - var w = TestObject(x: Zero) - check Protobuf.decode(Protobuf.encode(w), TestObject) == w - - var a = TestObject(y: some(One)) - check Protobuf.decode(Protobuf.encode(a), TestObject) == a - - var b = TestObject(z: some(@[One, NegOne, NegTwo, Zero])) - check Protobuf.decode(Protobuf.encode(b), TestObject) == b - - test "Option[Float] in object": - var x = FloatOption(x: some(1.5'f32)) - check Protobuf.decode(Protobuf.encode(x), FloatOption) == x - - var y = FloatOption(y: some(1.3'f64)) - check Protobuf.decode(Protobuf.encode(y), FloatOption) == y - - var z = FloatOption(x: some(1.5'f32), y: some(1.3'f64)) - check Protobuf.decode(Protobuf.encode(z), FloatOption) == z - - var v = FloatOption() - check Protobuf.decode(Protobuf.encode(v), FloatOption) == v - - test "Option[Fixed] in object": - var x = FixedOption(a: some(1'i32)) - check Protobuf.decode(Protobuf.encode(x), FixedOption) == x - - var y = FixedOption(b: some(1'i64)) - check Protobuf.decode(Protobuf.encode(y), FixedOption) == y - - var z = FixedOption(c: some(1'u32)) - check Protobuf.decode(Protobuf.encode(z), FixedOption) == z - - var v = FixedOption(d: some(1'u64)) - check Protobuf.decode(Protobuf.encode(v), FixedOption) == v - - #[ - This test has been commented for being pointless. - The reason this fails is because it detects a field number of 0, which is invalid. - Any valid field will be considered valid, as long as the length is correct. - If the length isn't, it's incorrect. - That said, those are two different things than remaining data. - test "Doesn't allow remaining data in the buffer": - expect ProtobufReadError: - discard Protobuf.decode(Protobuf.encode(SInt(5)) & @[byte(1)], type(SInt(int32))) - expect ProtobufReadError: - discard Protobuf.decode(Protobuf.encode(Basic(a: 100, b: "Test string.", c: 'C')) & @[byte(1)], type(Basic)) - ]# - - test "Doesn't allow unknown fields": - expect ProtobufMessageError: - discard Protobuf.decode((Protobuf.encode(Basic(a: 100, b: "Test string.", c: 'C')) & @[byte(4 shl 3)]), type(Basic)) + check Protobuf.decode(writer.getOutput(), type(Basic)) == Basic(b: repeated) diff --git a/tests/test_options.nim b/tests/test_options.nim deleted file mode 100644 index 0238c81..0000000 --- a/tests/test_options.nim +++ /dev/null @@ -1,139 +0,0 @@ -import options -import unittest - -import ../protobuf_serialization -from ../protobuf_serialization/internal import VarIntWrapped, FixedWrapped, unwrap, flatType, flatMap - -from test_objects import DistinctInt, `==` - -type - Basic = object - x {.sint, fieldNumber: 1.}: int32 - - Wrapped = object - y {.sint, fieldNumber: 1.}: Option[int32] - - Nested = ref object - child {.fieldNumber: 1.}: Option[Nested] - z {.fieldNumber: 2.}: Option[Wrapped] - -proc `==`*(lhs: Nested, rhs: Nested): bool = - lhs.z == rhs.z - -template testNone[T](ty: typedesc[T]) = - let output = Protobuf.encode(none(ty)) - check output.len == 0 - check Protobuf.decode(output, type(Option[T])).isNone() - -template testSome[T](value: T) = - let output = Protobuf.encode(some(value)) - check output == Protobuf.encode(flatMap(value)) - when flatType(T) is (VarIntWrapped or FixedWrapped): - check Protobuf.decode(output, type(Option[T])).get().unwrap() == some(value).get().unwrap() - else: - check Protobuf.decode(output, type(Option[T])) == some(value) - -suite "Test Encoding/Decoding of Options": - test "Option boolean": - testNone(bool) - testSome(true) - - test "Option signed VarInt": - testNone(PInt(int32)) - testSome(PInt(5'i32)) - testSome(PInt(-5'i32)) - - test "Option unsigned VarInt": - testNone(PInt(uint32)) - testSome(PInt(5'u32)) - - test "Option zigzagged VarInt": - testNone(SInt(int32)) - testSome(SInt(5'i32)) - testSome(SInt(-5'i32)) - - test "Option Fixed": - template fixedTest[T](value: T): untyped = - testNone(type(T)) - testSome(value) - - fixedTest(Fixed(5'i64)) - fixedTest(Fixed(-5'i64)) - fixedTest(Fixed(5'i32)) - fixedTest(Fixed(-5'i32)) - - fixedTest(Fixed(5'u64)) - fixedTest(Fixed(5'u32)) - - fixedTest(Float64(5.5'f64)) - fixedTest(Float64(-5.5'f64)) - fixedTest(Float32(5.5'f32)) - fixedTest(Float32(-5.5'f32)) - - test "Option length-delimited": - testNone(string) - testNone(seq[byte]) - - testSome("Testing string.") - testSome(@[byte(0), 1, 2, 3, 4]) - - test "Option object": - testNone(Basic) - testNone(Wrapped) - - testSome(Basic(x: 5'i32)) - testSome(Wrapped(y: some(5'i32))) - - test "Option ref": - #This is in a block, manually expanded, with a pointless initial value. - #Why? - #https://github.com/nim-lang/Nim/issues/14387 - block one4387: - var option = some(Nested()) - option = none(Nested) - - let output = Protobuf.encode(option) - check output.len == 0 - check Protobuf.decode(output, type(Option[Nested])).isNone() - - testSome(Nested( - child: some(Nested( - child: none(Nested), - z: none(Wrapped) - )), - z: none(Wrapped) - )) - - testSome(Nested( - child: none(Nested), - z: some(Wrapped(y: some(5'i32))) - )) - - testSome(Nested( - child: some(Nested( - child: none(Nested), - z: some(Wrapped(y: some(5'i32))) - )), - z: some(Wrapped(y: some(5'i32))) - )) - - testSome(Nested( - child: some(Nested( - z: some(Wrapped(y: some(5'i32))) - )), - z: some(Wrapped(y: some(5'i32))) - )) - - test "Option ptr": - testNone(ptr Basic) - - let basicInst = Basic(x: 5'i32) - let output = Protobuf.encode(some(basicInst)) - check output == Protobuf.encode(flatMap(basicInst)) - check Protobuf.decode(output, Option[ptr Basic]).get()[] == basicInst - - #This was banned at one point in this library's lifetime. - #It should work now. - test "Option Option": - testNone(string) - testSome(some("abc")) diff --git a/tests/test_protobuf2_semantics.nim b/tests/test_protobuf2_semantics.nim index 592eb56..912005b 100644 --- a/tests/test_protobuf2_semantics.nim +++ b/tests/test_protobuf2_semantics.nim @@ -1,22 +1,31 @@ -import options -import unittest +import unittest2 import ../protobuf_serialization type - Required {.protobuf2.} = object - a {.pint, fieldNumber: 1.}: PBOption[2'i32] - b {.pint, required, fieldNumber: 2.}: int32 + Required {.proto2.} = object + a {.fieldNumber: 1, pint .}: PBOption[2'i32] + b {.fieldNumber: 2, pint, required.}: int32 - FullOfDefaults {.protobuf2.} = object - a {.fieldNumber: 1.}: PBOption["abc"] - b {.fieldNumber: 2.}: Option[Required] + FullOfDefaults {.proto2.} = object + a {.fieldNumber: 3.}: PBOption["abc"] + b {.fieldNumber: 4.}: PBOption[default(Required)] - SeqContainer {.protobuf2.} = object - data {.fieldNumber: 1.}: seq[bool] + SeqContainer {.proto2.} = object + data {.fieldNumber: 5.}: seq[bool] - SeqString {.protobuf2.} = object - data {.fieldNumber: 1.}: seq[string] + SeqString {.proto2.} = object + data {.fieldNumber: 6.}: seq[string] + + FloatOption {.proto2.} = object + x {.fieldNumber: 1.}: PBOption[0'f32] + y {.fieldNumber: 2.}: PBOption[0'f64] + + FixedOption {.proto2.} = object + a {.fieldNumber: 1, fixed.}: PBOption[0'i32] + b {.fieldNumber: 2, fixed.}: PBOption[0'i64] + c {.fieldNumber: 3, fixed.}: PBOption[0'u32] + d {.fieldNumber: 4, fixed.}: PBOption[0'u64] suite "Test Encoding of Protobuf 2 Semantics": test "PBOption basics": @@ -40,11 +49,9 @@ suite "Test Encoding of Protobuf 2 Semantics": test "Requires required": expect ProtobufReadError: discard Protobuf.decode(@[], Required) - expect ProtobufReadError: - discard Protobuf.decode(Protobuf.encode(PInt(0'i32)), Required) test "Handles default": - var fod: FullOfDefaults = FullOfDefaults(b: some(Required(b: 5))) + var fod: FullOfDefaults = FullOfDefaults(b: PBOption[default(Required)].pbSome(Required(b: 5))) check: Protobuf.decode(Protobuf.encode(Required()), Required).a.isNone() Protobuf.decode(Protobuf.encode(Required()), Required).a.get() == 2 @@ -73,3 +80,31 @@ suite "Test Encoding of Protobuf 2 Semantics": check Protobuf.decode(Protobuf.encode(ssb), SeqString) == ssb ssb = SeqString(data: @["abc", "def", "ghi"]) check Protobuf.decode(Protobuf.encode(ssb), SeqString) == ssb + + test "Option[Float] in object": + var x = FloatOption(x: PBOption[0'f32].pbSome(1.5'f32)) + check Protobuf.decode(Protobuf.encode(x), FloatOption) == x + + var y = FloatOption(y: PBOption[0'f64].pbSome(1.3'f64)) + check Protobuf.decode(Protobuf.encode(y), FloatOption) == y + + var z = FloatOption( + x: PBOption[0'f32].pbSome(1.5'f32), + y: PBOption[0'f64].pbSome(1.3'f64)) + check Protobuf.decode(Protobuf.encode(z), FloatOption) == z + + var v = FloatOption() + check Protobuf.decode(Protobuf.encode(v), FloatOption) == v + + test "Option[Fixed] in object": + var x = FixedOption(a: PBOption[0'i32].pbSome(1'i32)) + check Protobuf.decode(Protobuf.encode(x), FixedOption) == x + + var y = FixedOption(b: PBOption[0'i64].pbSome(1'i64)) + check Protobuf.decode(Protobuf.encode(y), FixedOption) == y + + var z = FixedOption(c: PBOption[0'u32].pbSome(1'u32)) + check Protobuf.decode(Protobuf.encode(z), FixedOption) == z + + var v = FixedOption(d: PBOption[0'u64].pbSome(1'u64)) + check Protobuf.decode(Protobuf.encode(v), FixedOption) == v diff --git a/tests/test_repeated.nim b/tests/test_repeated.nim new file mode 100644 index 0000000..d2df262 --- /dev/null +++ b/tests/test_repeated.nim @@ -0,0 +1,60 @@ +import sets +import unittest2 +import stew/byteutils +import ../protobuf_serialization + +type + # Keep in sync with test_repeated.proto + Sequences {.proto3.} = object + x {.fieldNumber: 1, sint, packed: false.}: seq[int32] + y {.fieldNumber: 2, packed: false.}: seq[bool] + z {.fieldNumber: 3.}: seq[string] + + Packed {.proto3.} = object + x {.fieldNumber: 1, sint, packed: true.}: seq[int32] + y {.fieldNumber: 2, packed: true.}: seq[bool] + z {.fieldNumber: 3, fixed, packed: true.}: seq[int32] + a {.fieldNumber: 4, packed: true.}: seq[float32] + +suite "Test repeated fields": + test "Sequences": + # protoc --encode=Sequences test_repeated.proto | hexdump -ve '1/1 "%.2x"' + discard """ +x: [5, -3, 300, -612] +y: [true, false, true, true, false, false, false, true, false] +z: ["zero", "one", "two"] +""" + const + v = Sequences( + x: @[5'i32, -3, 300, -612], + y: @[true, false, true, true, false, false, false, true, false], + z: @["zero", "one", "two"] + ) + encoded = hexToSeqByte( + "080a080508d80408c7091001100010011001100010001000100110001a047a65726f1a036f6e651a0374776f") + + check: + Protobuf.encode(v) == encoded + Protobuf.decode(encoded, typeof(v)) == v + + test "Packed sequences": + # protoc --encode=Packed test_repeated.proto | hexdump -ve '1/1 "%.2x"' + discard """ +x: [5, -3, 300, -612] +y: [true, false, true, true, false, false, false, true, false] +z: [5, -3, 300, -612] +a: [5, -3, 300, -612] + """ + const + v = Packed( + x: @[5'i32, -3, 300, -612], + y: @[true, false, true, true, false, false, false, true, false], + z: @[5'i32, -3, 300, -612], + a: @[5'f32, -3, 300, -612], + ) + encoded = hexToSeqByte( + "0a060a05d804c70912090100010100000001001a1005000000fdffffff2c0100009cfdffff22100000a040000040c000009643000019c4") + + check: + Protobuf.encode(v) == encoded + Protobuf.decode(encoded, typeof(v)) == v diff --git a/tests/test_repeated.proto b/tests/test_repeated.proto new file mode 100644 index 0000000..02e41bb --- /dev/null +++ b/tests/test_repeated.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +message Sequences { + repeated sint32 x = 1 [packed = false]; + repeated bool y = 2 [packed = false]; + repeated string z = 3 [packed = false]; +} + +message Packed { + repeated sint32 x = 1 [packed = true]; + repeated bool y = 2 [packed = true]; + repeated sfixed32 z = 3 [packed = true]; + repeated float a = 4 [packed = true]; +} diff --git a/tests/test_stdlib.nim b/tests/test_stdlib.nim deleted file mode 100644 index 9de580a..0000000 --- a/tests/test_stdlib.nim +++ /dev/null @@ -1,88 +0,0 @@ -import sets -import unittest - -import ../protobuf_serialization -from ../protobuf_serialization/internal import unwrap - -type - Basic {.protobuf3.} = object - x {.pint, fieldNumber: 1.}: int32 - y {.fieldNumber: 2.}: seq[string] - - PragmadStdlib {.protobuf3.} = object - x {.sint, fieldNumber: 1.}: seq[int32] - #y {.pint, fieldNumber: 2.}: array[5, uint32] - z {.pfloat32, fieldNumber: 3.}: HashSet[float32] - - BooldStdlib {.protobuf3.} = object - x {.fieldNumber: 1.}: seq[bool] - #y {.fieldNumber: 2.}: array[3, bool] - -suite "Test Standard Lib Objects Encoding/Decoding": - #[test "Can encode/decode cstrings": - let str: cstring = "Testing string." - check Protobuf.decode(Protobuf.encode(str), type(cstring)) == str]# - - test "Can encode/decode seqs": - let - int64Seq = @[SInt(0'i64), SInt(-1'i64), SInt(1'i64), SInt(-1'i64)] - read = Protobuf.decode(Protobuf.encode(int64Seq), seq[SInt(int64)]) - check int64Seq.len == read.len - for i in 0 ..< int64Seq.len: - check int64Seq[i].unwrap() == read[i].unwrap() - - let basicSeq = @[ - Basic( - x: 0, - y: @[] - ), - Basic( - x: 1, - y: @["abc", "defg"] - ), - Basic( - x: 2, - y: @["hi", "jkl", "mnopq"] - ), - Basic( - x: -2, - y: @["xyz"] - ) - ] - check basicSeq == Protobuf.decode(Protobuf.encode(basicSeq), seq[Basic]) - - #[test "Can encode/decode arrays": - let - int64Arr = [SInt(0'i64), SInt(-1'i64), SInt(1'i64), SInt(-1'i64)] - read = Protobuf.decode(Protobuf.encode(int64Arr), type(seq[SInt(int64)])) - check int64Arr.len == read.len - for i in 0 ..< int64Arr.len: - check int64Arr[i].unwrap() == read[i].unwrap()]# - - test "Can encode/decode sets": - let - trueSet = {true} - falseSet = {false} - trueFalseSet = {true, false} - check Protobuf.decode(Protobuf.encode(trueSet), type(set[bool])) == trueSet - check Protobuf.decode(Protobuf.encode(falseSet), type(set[bool])) == falseSet - check Protobuf.decode(Protobuf.encode(trueFalseSet), type(set[bool])) == trueFalseSet - - test "Can encode/decode HashSets": - let setInstance = ["abc", "def", "ghi"].toHashSet() - check Protobuf.decode(Protobuf.encode(setInstance), type(HashSet[string])) == setInstance - - test "Can encode/decode stdlib fields where a pragma was used to specify encoding": - let pragmad = PragmadStdlib( - x: @[5'i32, -3'i32, 300'i32, -612'i32], - #y: [6'u32, 4'u32, 301'u32, 613'u32, 216'u32], - z: @[5.5'f32, 3.2'f32, 925.123].toHashSet() - ) - check Protobuf.decode(Protobuf.encode(pragmad), PragmadStdlib) == pragmad - - test "Can encode boolean seqs": #/arrays": - let boold = BooldStdlib( - x: @[true, false, true, true, false, false, false, true, false], - #y: [true, true, false] - ) - check Protobuf.decode(Protobuf.encode(boold), BooldStdlib) == boold diff --git a/tests/test_thirty_three_fields.nim b/tests/test_thirty_three_fields.nim index 78d4f61..ba8f373 100644 --- a/tests/test_thirty_three_fields.nim +++ b/tests/test_thirty_three_fields.nim @@ -1,8 +1,8 @@ -import unittest +import unittest2 import ../protobuf_serialization -type X {.protobuf3.} = object +type X {.proto3.} = object x00 {.fieldNumber: 1.}: bool x01 {.fieldNumber: 2.}: bool x02 {.fieldNumber: 3.}: bool