Signed variable integers fixes. (#96)

* Fix signed varints.
Add tests for signed varints.
Remove some casts to allow usage at compile time.

* Fix vsizeof() on 32bit platforms.

* Add `hint` and `zint` types for proper signed integer encoding.

* Fix varint related bugs.

* Update requirements.

* Fix interop tests because of fixed readLine.

* Add putVarint, getVarint and tests.
This commit is contained in:
Eugene Kabanov 2020-03-06 21:19:43 +02:00 committed by GitHub
parent 381630f185
commit 5701d937c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 399 additions and 58 deletions

View File

@ -10,8 +10,8 @@ skipDirs = @["tests", "examples", "Nim"]
requires "nim > 0.19.4",
"secp256k1",
"nimcrypto >= 0.4.1",
"chronos >= 2.3.5",
"bearssl >= 0.1.3",
"chronos >= 2.3.8",
"bearssl >= 0.1.4",
"chronicles >= 0.7.0",
"stew"

View File

@ -8,8 +8,8 @@
## those terms.
## This module implementes API for `go-libp2p-daemon`.
import os, osproc, strutils, tables, streams, strtabs
import chronos, stew/base58
import os, osproc, strutils, tables, strtabs
import chronos
import ../varint, ../multiaddress, ../multicodec, ../cid, ../peer
import ../wire, ../multihash, ../protobuf/minprotobuf
import ../crypto/crypto
@ -175,7 +175,7 @@ proc requestConnect(peerid: PeerID,
for item in addresses:
msg.write(initProtoField(2, item.data.buffer))
if timeout > 0:
msg.write(initProtoField(3, timeout))
msg.write(initProtoField(3, hint64(timeout)))
result.write(initProtoField(1, cast[uint](RequestType.CONNECT)))
result.write(initProtoField(2, msg))
result.finish()
@ -201,7 +201,7 @@ proc requestStreamOpen(peerid: PeerID,
for item in protocols:
msg.write(initProtoField(2, item))
if timeout > 0:
msg.write(initProtoField(3, timeout))
msg.write(initProtoField(3, hint64(timeout)))
result.write(initProtoField(1, cast[uint](RequestType.STREAM_OPEN)))
result.write(initProtoField(3, msg))
result.finish()
@ -235,7 +235,7 @@ proc requestDHTFindPeer(peer: PeerID, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid))
msg.write(initProtoField(2, peer))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -251,7 +251,7 @@ proc requestDHTFindPeersConnectedToPeer(peer: PeerID,
msg.write(initProtoField(1, msgid))
msg.write(initProtoField(2, peer))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -268,7 +268,7 @@ proc requestDHTFindProviders(cid: Cid,
msg.write(initProtoField(3, cid.data.buffer))
msg.write(initProtoField(6, count))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -283,7 +283,7 @@ proc requestDHTGetClosestPeers(key: string, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid))
msg.write(initProtoField(4, key))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -298,7 +298,7 @@ proc requestDHTGetPublicKey(peer: PeerID, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid))
msg.write(initProtoField(2, peer))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -313,7 +313,7 @@ proc requestDHTGetValue(key: string, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid))
msg.write(initProtoField(4, key))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -328,7 +328,7 @@ proc requestDHTSearchValue(key: string, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid))
msg.write(initProtoField(4, key))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -345,7 +345,7 @@ proc requestDHTPutValue(key: string, value: openarray[byte],
msg.write(initProtoField(4, key))
msg.write(initProtoField(5, value))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -360,7 +360,7 @@ proc requestDHTProvide(cid: Cid, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid))
msg.write(initProtoField(3, cid.data.buffer))
if timeout > 0:
msg.write(initProtoField(7, uint(timeout)))
msg.write(initProtoField(7, hint64(timeout)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg))
@ -374,7 +374,7 @@ proc requestCMTagPeer(peer: PeerID, tag: string, weight: int): ProtoBuffer =
msg.write(initProtoField(1, msgid))
msg.write(initProtoField(2, peer))
msg.write(initProtoField(3, tag))
msg.write(initProtoField(4, weight))
msg.write(initProtoField(4, hint64(weight)))
msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.CONNMANAGER)))
result.write(initProtoField(6, msg))
@ -949,7 +949,6 @@ proc listPeers*(api: DaemonAPI): Future[seq[PeerInfo]] {.async.} =
try:
var pb = await transp.transactMessage(requestListPeers())
pb.withMessage() do:
var address = newSeq[byte]()
result = newSeq[PeerInfo]()
var res = pb.enterSubmessage()
while res != 0:

View File

@ -150,7 +150,7 @@ proc encodeSubs*(subs: SubOpts, pb: var ProtoBuffer) {.gcsafe.} =
proc decodeSubs*(pb: var ProtoBuffer): seq[SubOpts] {.gcsafe.} =
while true:
var subOpt: SubOpts
var subscr: int
var subscr: uint
discard pb.getVarintValue(1, subscr)
subOpt.subscribe = cast[bool](subscr)
trace "read subscribe field", subscribe = subOpt.subscribe

View File

@ -15,7 +15,7 @@
## - LibP2P varint, which is able to encode only 63bits of uint64 number and
## maximum size of encoded value is 9 octets (bytes).
## https://github.com/multiformats/unsigned-varint
import bitops
import bitops, typetraits
type
VarintStatus* {.pure.} = enum
@ -31,21 +31,70 @@ type
LP* = object
## Use this type to specify LibP2P varint encoding
zint32* = distinct int32
zint64* = distinct int64
zint* = distinct int
## Signed integer types which will be encoded using zigzag encoding.
hint32* = distinct int32
hint64* = distinct int64
hint* = distinct int
## Signed integer types which will be encoded using simple cast.
PBSomeUVarint* = uint | uint64 | uint32
PBSomeSVarint* = int | int64 | int32
PBSomeVarint* = PBSomeUVarint | PBSomeSVarint
PBSomeSVarint* = hint | hint64 | hint32
PBZigVarint* = zint | zint64 | zint32
PBSomeVarint* = PBSomeUVarint | PBSomeSVarint | PBZigVarint
LPSomeUVarint* = uint | uint64 | uint32 | uint16 | uint8
LPSomeVarint* = LPSomeUVarint
SomeVarint* = PBSomeVarint | LPSomeVarint
SomeUVarint* = PBSomeUVarint | LPSomeUVarint
VarintError* = object of CatchableError
proc vsizeof*(x: SomeVarint): int {.inline.} =
proc vsizeof*(x: SomeUVarint): int {.inline.} =
## Returns number of bytes required to encode integer ``x`` as varint.
if x == cast[type(x)](0):
result = 1
if x == type(x)(0):
1
else:
result = (fastLog2(x) + 1 + 7 - 1) div 7
(fastLog2(x) + 1 + 7 - 1) div 7
proc vsizeof*(x: PBSomeSVarint): int {.inline.} =
## Returns number of bytes required to encode signed integer ``x``.
##
## Note: This procedure interprets signed integer as ProtoBuffer's
## ``int32`` and ``int64`` integers.
when sizeof(x) == 8:
if int64(x) == 0'i64:
1
else:
(fastLog2(uint64(x)) + 1 + 7 - 1) div 7
else:
if int32(x) == 0'i32:
1
else:
(fastLog2(uint32(x)) + 1 + 7 - 1) div 7
proc vsizeof*(x: PBZigVarint): int {.inline.} =
## Returns number of bytes required to encode signed integer ``x``.
##
## Note: This procedure interprets signed integer as ProtoBuffer's
## ``sint32`` and ``sint64`` integer.
when sizeof(x) == 8:
if int64(x) == 0'i64:
1
else:
if int64(x) < 0'i64:
vsizeof(not(uint64(x) shl 1))
else:
vsizeof(uint64(x) shl 1)
else:
if int32(x) == 0'i32:
1
else:
if int32(x) < 0'i32:
vsizeof(not(uint32(x) shl 1))
else:
vsizeof(uint32(x) shl 1)
proc getUVarint*[T: PB|LP](vtype: typedesc[T],
pbytes: openarray[byte],
@ -83,16 +132,16 @@ proc getUVarint*[T: PB|LP](vtype: typedesc[T],
var shift = 0'u8
result = VarintStatus.Incomplete
outlen = 0
outval = cast[type(outval)](0)
outval = type(outval)(0)
for i in 0..<len(pbytes):
let b = pbytes[i]
if shift >= MaxBits:
result = VarintStatus.Overflow
outlen = 0
outval = cast[type(outval)](0)
outval = type(outval)(0)
break
else:
outval = outval or (cast[type(outval)](b and 0x7F'u8) shl shift)
outval = outval or (type(outval)(b and 0x7F'u8) shl shift)
shift += 7
inc(outlen)
if (b and 0x80'u8) == 0'u8:
@ -100,12 +149,12 @@ proc getUVarint*[T: PB|LP](vtype: typedesc[T],
break
if result == VarintStatus.Incomplete:
outlen = 0
outval = cast[type(outval)](0)
outval = type(outval)(0)
when vtype is LP:
if result == VarintStatus.Success:
if outlen != vsizeof(outval):
outval = cast[type(outval)](0)
outval = type(outval)(0)
outlen = 0
result = VarintStatus.Overlong
@ -135,16 +184,16 @@ proc putUVarint*[T: PB|LP](vtype: typedesc[T],
when vtype is LP:
if sizeof(outval) == 8:
if (cast[uint64](outval) and 0x8000_0000_0000_0000'u64) != 0'u64:
if (uint64(outval) and 0x8000_0000_0000_0000'u64) != 0'u64:
result = Overflow
return
if value <= cast[type(outval)](0x7F):
buffer[0] = cast[byte](outval and 0xFF)
if value <= type(outval)(0x7F):
buffer[0] = byte(outval and 0xFF)
inc(k)
else:
while value != cast[type(outval)](0):
buffer[k] = cast[byte]((value and 0x7F) or 0x80)
while value != type(outval)(0):
buffer[k] = byte((value and 0x7F) or 0x80)
value = value shr 7
inc(k)
buffer[k - 1] = buffer[k - 1] and 0x7F'u8
@ -158,9 +207,11 @@ proc putUVarint*[T: PB|LP](vtype: typedesc[T],
proc getSVarint*(pbytes: openarray[byte], outsize: var int,
outval: var PBSomeSVarint): VarintStatus {.inline.} =
## Decode Google ProtoBuf's `signed varint` from buffer ``pbytes`` and store
## it to ``outval``. On success ``outlen`` will be set to number of bytes
## processed while decoding `signed varint`.
## Decode signed integer (``int32`` or ``int64``) from buffer ``pbytes``
## and store it to ``outval``.
##
## On success ``outlen`` will be set to number of bytes processed while
## decoding signed varint.
##
## If array ``pbytes`` is empty, ``Incomplete`` error will be returned.
##
@ -181,15 +232,44 @@ proc getSVarint*(pbytes: openarray[byte], outsize: var int,
result = PB.getUVarint(pbytes, outsize, value)
if result == VarintStatus.Success:
if (value and cast[type(value)](1)) != cast[type(value)](0):
outval = cast[type(outval)](value)
proc getSVarint*(pbytes: openarray[byte], outsize: var int,
outval: var PBZigVarint): VarintStatus {.inline.} =
## Decode Google ProtoBuf's zigzag encoded signed integer (``sint32`` or
## ``sint64`` ) from buffer ``pbytes`` and store it to ``outval``.
##
## On success ``outlen`` will be set to number of bytes processed while
## decoding signed varint.
##
## If array ``pbytes`` is empty, ``Incomplete`` error will be returned.
##
## If there not enough bytes available in array ``pbytes`` to decode `signed
## varint`, ``Incomplete`` error will be returned.
##
## If encoded value can produce integer overflow, ``Overflow`` error will be
## returned.
##
## Note, when decoding 10th byte of 64bit integer only 1 bit from byte will be
## decoded, all other bits will be ignored. When decoding 5th byte of 32bit
## integer only 4 bits from byte will be decoded, all other bits will be
## ignored.
when sizeof(outval) == 8:
var value: uint64
else:
var value: uint32
result = PB.getUVarint(pbytes, outsize, value)
if result == VarintStatus.Success:
if (value and type(value)(1)) != type(value)(0):
outval = cast[type(outval)](not(value shr 1))
else:
outval = cast[type(outval)](value shr 1)
proc putSVarint*(pbytes: var openarray[byte], outsize: var int,
outval: PBSomeSVarint): VarintStatus {.inline.} =
## Encode Google ProtoBuf's `signed varint` ``outval`` and store it to array
## ``pbytes``.
outval: PBZigVarint): VarintStatus {.inline.} =
## Encode signed integer ``outval`` using ProtoBuffer's zigzag encoding
## (``sint32`` or ``sint64``) and store it to array ``pbytes``.
##
## On success ``outlen`` will hold number of bytes (octets) used to encode
## unsigned integer ``v``.
@ -202,18 +282,76 @@ proc putSVarint*(pbytes: var openarray[byte], outsize: var int,
## Maximum encoded length of 32bit integer is 5 octets.
when sizeof(outval) == 8:
var value: uint64 =
if outval < 0:
not(cast[uint64](outval) shl 1)
if int64(outval) < 0'i64:
not(uint64(outval) shl 1)
else:
cast[uint64](outval) shl 1
uint64(outval) shl 1
else:
var value: uint32 =
if outval < 0:
not(cast[uint32](outval) shl 1)
if int32(outval) < 0'i32:
not(uint32(outval) shl 1)
else:
cast[uint32](outval) shl 1
uint32(outval) shl 1
result = PB.putUVarint(pbytes, outsize, value)
proc putSVarint*(pbytes: var openarray[byte], outsize: var int,
outval: PBSomeSVarint): VarintStatus {.inline.} =
## Encode signed integer ``outval`` (``int32`` or ``int64``) and store it to
## array ``pbytes``.
##
## On success ``outlen`` will hold number of bytes (octets) used to encode
## unsigned integer ``v``.
##
## If there not enough bytes available in buffer ``pbytes``, ``Incomplete``
## error will be returned and ``outlen`` will be set to number of bytes
## required.
##
## Maximum encoded length of 64bit integer is 10 octets.
## Maximum encoded length of 32bit integer is 5 octets.
when sizeof(outval) == 8:
result = PB.putUVarint(pbytes, outsize, uint64(outval))
else:
result = PB.putUVarint(pbytes, outsize, uint32(outval))
template varintFatal(msg) =
const m = msg
{.fatal: m.}
proc putVarint*[T: PB|LP](vtype: typedesc[T], pbytes: var openarray[byte],
nbytes: var int, value: SomeVarint): VarintStatus {.inline.} =
when vtype is PB:
when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
result = putSVarint(pbytes, nbytes, value)
elif (type(value) is PBSomeUVarint):
result = PB.putUVarint(pbytes, nbytes, value)
else:
varintFatal("Protobuf's varint do not support type [" &
typetraits.name(type(value)) & "]")
elif vtype is LP:
when (type(value) is LPSomeVarint):
result = LP.putUVarint(pbytes, nbytes, value)
else:
varintFatal("LibP2P's varint do not support type [" &
typetraits.name(type(value)) & "]")
proc getVarint*[T: PB|LP](vtype: typedesc[T], pbytes: openarray[byte],
nbytes: var int,
value: var SomeVarint): VarintStatus {.inline.} =
when vtype is PB:
when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
result = getSVarint(pbytes, nbytes, value)
elif (type(value) is PBSomeUVarint):
result = PB.getUVarint(pbytes, nbytes, value)
else:
varintFatal("Protobuf's varint do not support type [" &
typetraits.name(type(value)) & "]")
elif vtype is LP:
when (type(value) is LPSomeVarint):
result = LP.getUVarint(pbytes, nbytes, value)
else:
varintFatal("LibP2P's varint do not support type [" &
typetraits.name(type(value)) & "]")
proc encodeVarint*(vtype: typedesc[PB],
value: PBSomeVarint): seq[byte] {.inline.} =
## Encode integer to Google ProtoBuf's `signed/unsigned varint` and returns
@ -224,7 +362,7 @@ proc encodeVarint*(vtype: typedesc[PB],
result.setLen(5)
else:
result.setLen(10)
when type(value) is PBSomeSVarint:
when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
let res = putSVarint(result, outsize, value)
else:
let res = PB.putUVarint(result, outsize, value)

View File

@ -74,14 +74,14 @@ proc writeLPVarint*(vb: var VBuffer, value: LPSomeUVarint) =
doAssert(res == VarintStatus.Success)
vb.offset += length
proc writeVarint*(vb: var VBuffer, value: LPSomeUVarint) =
proc writeVarint*(vb: var VBuffer, value: LPSomeUVarint) =
writeLPVarint(vb, value)
proc writeSeq*[T: byte|char](vb: var VBuffer, value: openarray[T]) =
## Write array ``value`` to buffer ``vb``, value will be prefixed with
## varint length of the array.
var length = 0
vb.buffer.setLen(len(vb.buffer) + vsizeof(len(value)) + len(value))
vb.buffer.setLen(len(vb.buffer) + vsizeof(uint(len(value))) + len(value))
let res = LP.putUVarint(toOpenArray(vb.buffer, vb.offset, len(vb.buffer) - 1),
length, uint(len(value)))
doAssert(res == VarintStatus.Success)

View File

@ -232,6 +232,14 @@ suite "Interop":
proc runTests(): Future[bool] {.async.} =
var protos = @["/test-stream"]
var test = "TEST STRING"
# We are preparing expect string, which should be prefixed with varint
# length and do not have `\r\n` suffix, because we going to use
# readLine().
var buffer = initVBuffer()
buffer.writeSeq(test & "\r\n")
buffer.finish()
var expect = newString(len(buffer) - 2)
copyMem(addr expect[0], addr buffer.buffer[0], len(expect))
let nativeNode = createNode()
let awaiters = await nativeNode.start()
@ -241,8 +249,10 @@ suite "Interop":
var testFuture = newFuture[string]("test.future")
proc daemonHandler(api: DaemonAPI, stream: P2PStream) {.async.} =
# We should perform `readLp()` instead of `readLine()`. `readLine()`
# here reads actually length prefixed string.
var line = await stream.transp.readLine()
check line == test
check line == expect
testFuture.complete(line)
await daemonNode.addHandler(protos, daemonHandler)
@ -250,7 +260,7 @@ suite "Interop":
daemonPeer.addresses),
protos[0])
await conn.writeLp(test & "\r\n")
result = test == (await wait(testFuture, 10.secs))
result = expect == (await wait(testFuture, 10.secs))
await nativeNode.stop()
await allFutures(awaiters)
await daemonNode.close()

View File

@ -16,6 +16,84 @@ const PBedgeValues = [
(1'u64 shl 63), 0xFFFF_FFFF_FFFF_FFFF'u64
]
const PBPositiveSignedEdgeValues = [
0'u64, 0x3F'u64,
0x40'u64, 0x1FFF'u64,
0x2000'u64, 0xFFFFF'u64,
0x100000'u64, 0x7FFFFFF'u64,
0x8000000'u64, 0x3FFFFFFFF'u64,
0x400000000'u64, 0x1FFFFFFFFFF'u64,
0x20000000000'u64, 0xFFFFFFFFFFFF'u64,
0x1000000000000'u64, 0x7FFFFFFFFFFFFF'u64,
0x80000000000000'u64, 0x3FFFFFFFFFFFFFFF'u64,
0x4000000000000000'u64, 0x7FFFFFFFFFFFFFFF'u64
]
const PBNegativeSignedEdgeValues = [
0x0000000000000000'u64, 0xFFFFFFFFFFFFFFC0'u64,
0xFFFFFFFFFFFFFFBF'u64, 0xFFFFFFFFFFFFE000'u64,
0xFFFFFFFFFFFFDFFF'u64, 0xFFFFFFFFFFF00000'u64,
0xFFFFFFFFFFEFFFFF'u64, 0xFFFFFFFFF8000000'u64,
0xFFFFFFFFF7FFFFFF'u64, 0xFFFFFFFC00000000'u64,
0xFFFFFFFBFFFFFFFF'u64, 0xFFFFFE0000000000'u64,
0xFFFFFDFFFFFFFFFF'u64, 0xFFFF000000000000'u64,
0xFFFEFFFFFFFFFFFF'u64, 0xFF80000000000000'u64,
0xFF7FFFFFFFFFFFFF'u64, 0xC000000000000000'u64,
0xBFFFFFFFFFFFFFFF'u64, 0x8000000000000000'u64
]
const PBPositiveSignedZigZagEdgeExpects = [
"00", "7E",
"8001", "FE7F",
"808001", "FEFF7F",
"80808001", "FEFFFF7F",
"8080808001", "FEFFFFFF7F",
"808080808001", "FEFFFFFFFF7F",
"80808080808001", "FEFFFFFFFFFF7F",
"8080808080808001", "FEFFFFFFFFFFFF7F",
"808080808080808001", "FEFFFFFFFFFFFFFF7F",
"80808080808080808001", "FEFFFFFFFFFFFFFFFF01"
]
const PBNegativeSignedZigZagEdgeExpects = [
"00", "7F",
"8101", "FF7F",
"818001", "FFFF7F",
"81808001", "FFFFFF7F",
"8180808001", "FFFFFFFF7F",
"818080808001", "FFFFFFFFFF7F",
"81808080808001", "FFFFFFFFFFFF7F",
"8180808080808001", "FFFFFFFFFFFFFF7F",
"818080808080808001", "FFFFFFFFFFFFFFFF7F",
"81808080808080808001", "FFFFFFFFFFFFFFFFFF01",
]
const PBPositiveSignedEdgeExpects = [
"00", "3F",
"40", "FF3F",
"8040", "FFFF3F",
"808040", "FFFFFF3F",
"80808040", "FFFFFFFF3F",
"8080808040", "FFFFFFFFFF3F",
"808080808040", "FFFFFFFFFFFF3F",
"80808080808040", "FFFFFFFFFFFFFF3F",
"8080808080808040", "FFFFFFFFFFFFFFFF3F",
"808080808080808040", "FFFFFFFFFFFFFFFF7F"
]
const PBNegativeSignedEdgeExpects = [
"00", "C0FFFFFFFFFFFFFFFF01",
"BFFFFFFFFFFFFFFFFF01", "80C0FFFFFFFFFFFFFF01",
"FFBFFFFFFFFFFFFFFF01", "8080C0FFFFFFFFFFFF01",
"FFFFBFFFFFFFFFFFFF01", "808080C0FFFFFFFFFF01",
"FFFFFFBFFFFFFFFFFF01", "80808080C0FFFFFFFF01",
"FFFFFFFFBFFFFFFFFF01", "8080808080C0FFFFFF01",
"FFFFFFFFFFBFFFFFFF01", "808080808080C0FFFF01",
"FFFFFFFFFFFFBFFFFF01", "80808080808080C0FF01",
"FFFFFFFFFFFFFFBFFF01", "8080808080808080C001",
"FFFFFFFFFFFFFFFFBF01", "80808080808080808001"
]
const PBedgeExpects = [
"00", "7F",
"8001", "FF7F",
@ -33,6 +111,22 @@ const PBedgeSizes = [
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10
]
const PBEdgeSignedPositiveZigZagSizes = [
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10
]
const PBEdgeSignedNegativeZigZagSizes = [
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10
]
const PBEdgeSignedPositiveSizes = [
1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9
]
const PBEdgeSignedNegativeSizes = [
1, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10
]
const LPedgeValues = [
0'u64, (1'u64 shl 7) - 1'u64,
(1'u64 shl 7), (1'u64 shl 14) - 1'u64,
@ -85,21 +179,72 @@ proc toHex*(a: openarray[byte], lowercase: bool = false): string =
suite "Variable integer test suite":
test "vsizeof() edge cases test":
for i in 0..<len(PBedgeValues):
for i in 0 ..< len(PBedgeValues):
check vsizeof(PBedgeValues[i]) == PBedgeSizes[i]
for i in 0 ..< len(PBPositiveSignedEdgeValues):
check:
vsizeof(hint64(PBPositiveSignedEdgeValues[i])) ==
PBEdgeSignedPositiveSizes[i]
vsizeof(zint64(PBPositiveSignedEdgeValues[i])) ==
PBEdgeSignedPositiveZigZagSizes[i]
for i in 0 ..< len(PBNegativeSignedEdgeValues):
check:
vsizeof(hint64(PBNegativeSignedEdgeValues[i])) ==
PBEdgeSignedNegativeSizes[i]
vsizeof(zint64(PBNegativeSignedEdgeValues[i])) ==
PBEdgeSignedNegativeZigZagSizes[i]
test "[ProtoBuf] Success edge cases test":
var buffer = newSeq[byte]()
var length = 0
var value = 0'u64
for i in 0..<len(PBedgeValues):
var uvalue = 0'u64
var ivalue = hint64(0)
var svalue = zint64(0)
for i in 0 ..< len(PBedgeValues):
buffer.setLen(PBedgeSizes[i])
check:
PB.putUVarint(buffer, length, PBedgeValues[i]) == VarintStatus.Success
PB.getUVarint(buffer, length, value) == VarintStatus.Success
value == PBedgeValues[i]
PB.getUVarint(buffer, length, uvalue) == VarintStatus.Success
uvalue == PBedgeValues[i]
toHex(buffer) == PBedgeExpects[i]
for i in 0 ..< len(PBPositiveSignedEdgeValues):
buffer.setLen(PBEdgeSignedPositiveSizes[i])
check:
putSVarint(buffer, length,
hint64(PBPositiveSignedEdgeValues[i])) == VarintStatus.Success
getSVarint(buffer, length, ivalue) == VarintStatus.Success
int64(ivalue) == int64(PBPositiveSignedEdgeValues[i])
toHex(buffer) == PBPositiveSignedEdgeExpects[i]
buffer.setLen(PBEdgeSignedPositiveZigZagSizes[i])
check:
putSVarint(buffer, length,
zint64(PBPositiveSignedEdgeValues[i])) == VarintStatus.Success
getSVarint(buffer, length, svalue) == VarintStatus.Success
int64(svalue) == int64(PBPositiveSignedEdgeValues[i])
toHex(buffer) == PBPositiveSignedZigZagEdgeExpects[i]
for i in 0 ..< len(PBNegativeSignedEdgeValues):
buffer.setLen(PBEdgeSignedNegativeSizes[i])
check:
putSVarint(buffer, length,
hint64(PBNegativeSignedEdgeValues[i])) == VarintStatus.Success
getSVarint(buffer, length, ivalue) == VarintStatus.Success
int64(ivalue) == int64(PBNegativeSignedEdgeValues[i])
toHex(buffer) == PBNegativeSignedEdgeExpects[i]
buffer.setLen(PBEdgeSignedNegativeZigZagSizes[i])
check:
putSVarint(buffer, length,
zint64(PBNegativeSignedEdgeValues[i])) == VarintStatus.Success
getSVarint(buffer, length, svalue) == VarintStatus.Success
int64(svalue) == int64(PBNegativeSignedEdgeValues[i])
toHex(buffer) == PBNegativeSignedZigZagEdgeExpects[i]
test "[ProtoBuf] Buffer Overrun edge cases test":
var buffer = newSeq[byte]()
var length = 0
@ -147,6 +292,23 @@ suite "Variable integer test suite":
check:
PB.getUVarint(buffer, length, value) == VarintStatus.Overflow
test "[ProtoBuf] Test vectors":
# The test vectors which was obtained at:
# https://github.com/dermesser/integer-encoding-rs/blob/master/src/varint_tests.rs
# https://github.com/That3Percent/zigzag/blob/master/src/lib.rs
check:
PB.encodeVarint(0'u64) == @[0x00'u8]
PB.encodeVarint(0'u32) == @[0x00'u8]
PB.encodeVarint(hint64(0)) == @[0x00'u8]
PB.encodeVarint(hint32(0)) == @[0x00'u8]
PB.encodeVarint(zint64(0)) == @[0x00'u8]
PB.encodeVarint(zint32(0)) == @[0x00'u8]
PB.encodeVarint(zint32(-1)) == PB.encodeVarint(1'u32)
PB.encodeVarint(zint64(150)) == PB.encodeVarint(300'u32)
PB.encodeVarint(zint64(-150)) == PB.encodeVarint(299'u32)
PB.encodeVarint(zint32(-2147483648)) == PB.encodeVarint(4294967295'u64)
PB.encodeVarint(zint32(2147483647)) == PB.encodeVarint(4294967294'u64)
test "[LibP2P] Success edge cases test":
var buffer = newSeq[byte]()
var length = 0
@ -267,3 +429,35 @@ suite "Variable integer test suite":
LP.getUVarint(@[0x80'u8, 0x00'u8], length, value) == VarintStatus.Overlong
length == 0
value == 0
test "getVarint/putVarint tests":
proc `==`(a, b: zint32|hint32): bool =
int32(a) == int32(b)
proc `==`(a, b: zint64|hint64): bool =
int64(a) == int64(b)
template varintTest(ttype, vtype, value, expect: untyped) =
var ovalue: vtype
var buffer = newSeq[byte](10)
var length = 0
check ttype.putVarint(buffer, length, value) == VarintStatus.Success
buffer.setLen(length)
check:
toHex(buffer) == expect
ttype.getVarint(buffer, length, ovalue) == VarintStatus.Success
ovalue == value
varintTest(PB, uint64, high(uint64), "FFFFFFFFFFFFFFFFFF01")
varintTest(PB, uint32, high(uint32), "FFFFFFFF0F")
varintTest(PB, zint64, zint64(high(int64)), "FEFFFFFFFFFFFFFFFF01")
varintTest(PB, zint32, zint32(high(int32)), "FEFFFFFF0F")
varintTest(PB, zint64, zint64(low(int64)), "FFFFFFFFFFFFFFFFFF01")
varintTest(PB, zint32, zint32(low(int32)), "FFFFFFFF0F")
varintTest(PB, hint64, hint64(high(int64)), "FFFFFFFFFFFFFFFF7F")
varintTest(PB, hint32, hint32(high(int32)), "FFFFFFFF07")
varintTest(PB, hint64, hint64(low(int64)), "80808080808080808001")
varintTest(PB, hint32, hint32(low(int32)), "8080808008")
varintTest(LP, uint64, uint64(high(int64)), "FFFFFFFFFFFFFFFF7F")
varintTest(LP, uint32, uint32(high(uint32)), "FFFFFFFF0F")
varintTest(LP, uint16, uint16(high(uint16)), "FFFF03")
varintTest(LP, uint8, uint8(high(uint8)), "FF01")