codex-storage
/
nim-libp2p
mirror of https://github.com/codex-storage/nim-libp2p.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Signed variable integers fixes. (#96) 

* Fix signed varints.
Add tests for signed varints.
Remove some casts to allow usage at compile time.

* Fix vsizeof() on 32bit platforms.

* Add `hint` and `zint` types for proper signed integer encoding.

* Fix varint related bugs.

* Update requirements.

* Fix interop tests because of fixed readLine.

* Add putVarint, getVarint and tests.
This commit is contained in:
Eugene Kabanov 2020-03-06 21:19:43 +02:00 committed by GitHub
parent 381630f185
commit 5701d937c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 399 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libp2p.nimble
View File

@ -10,8 +10,8 @@ skipDirs = @["tests", "examples", "Nim"]
requires "nim > 0.19.4", requires "nim > 0.19.4",
"secp256k1", "secp256k1",
"nimcrypto >= 0.4.1", "nimcrypto >= 0.4.1",
"chronos >= 2.3.5", "chronos >= 2.3.8",
"bearssl >= 0.1.3", "bearssl >= 0.1.4",
"chronicles >= 0.7.0", "chronicles >= 0.7.0",
"stew" "stew"

29
libp2p/daemon/daemonapi.nim
View File

@ -8,8 +8,8 @@
## those terms. ## those terms.
## This module implementes API for `go-libp2p-daemon`. ## This module implementes API for `go-libp2p-daemon`.
import os, osproc, strutils, tables, streams, strtabs import os, osproc, strutils, tables, strtabs
import chronos, stew/base58 import chronos
import ../varint, ../multiaddress, ../multicodec, ../cid, ../peer import ../varint, ../multiaddress, ../multicodec, ../cid, ../peer
import ../wire, ../multihash, ../protobuf/minprotobuf import ../wire, ../multihash, ../protobuf/minprotobuf
import ../crypto/crypto import ../crypto/crypto
@ -175,7 +175,7 @@ proc requestConnect(peerid: PeerID,
for item in addresses: for item in addresses:
msg.write(initProtoField(2, item.data.buffer)) msg.write(initProtoField(2, item.data.buffer))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(3, timeout)) msg.write(initProtoField(3, hint64(timeout)))
result.write(initProtoField(1, cast[uint](RequestType.CONNECT))) result.write(initProtoField(1, cast[uint](RequestType.CONNECT)))
result.write(initProtoField(2, msg)) result.write(initProtoField(2, msg))
result.finish() result.finish()
@ -201,7 +201,7 @@ proc requestStreamOpen(peerid: PeerID,
for item in protocols: for item in protocols:
msg.write(initProtoField(2, item)) msg.write(initProtoField(2, item))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(3, timeout)) msg.write(initProtoField(3, hint64(timeout)))
result.write(initProtoField(1, cast[uint](RequestType.STREAM_OPEN))) result.write(initProtoField(1, cast[uint](RequestType.STREAM_OPEN)))
result.write(initProtoField(3, msg)) result.write(initProtoField(3, msg))
result.finish() result.finish()
@ -235,7 +235,7 @@ proc requestDHTFindPeer(peer: PeerID, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid)) msg.write(initProtoField(1, msgid))
msg.write(initProtoField(2, peer)) msg.write(initProtoField(2, peer))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -251,7 +251,7 @@ proc requestDHTFindPeersConnectedToPeer(peer: PeerID,
msg.write(initProtoField(1, msgid)) msg.write(initProtoField(1, msgid))
msg.write(initProtoField(2, peer)) msg.write(initProtoField(2, peer))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -268,7 +268,7 @@ proc requestDHTFindProviders(cid: Cid,
msg.write(initProtoField(3, cid.data.buffer)) msg.write(initProtoField(3, cid.data.buffer))
msg.write(initProtoField(6, count)) msg.write(initProtoField(6, count))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -283,7 +283,7 @@ proc requestDHTGetClosestPeers(key: string, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid)) msg.write(initProtoField(1, msgid))
msg.write(initProtoField(4, key)) msg.write(initProtoField(4, key))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -298,7 +298,7 @@ proc requestDHTGetPublicKey(peer: PeerID, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid)) msg.write(initProtoField(1, msgid))
msg.write(initProtoField(2, peer)) msg.write(initProtoField(2, peer))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -313,7 +313,7 @@ proc requestDHTGetValue(key: string, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid)) msg.write(initProtoField(1, msgid))
msg.write(initProtoField(4, key)) msg.write(initProtoField(4, key))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -328,7 +328,7 @@ proc requestDHTSearchValue(key: string, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid)) msg.write(initProtoField(1, msgid))
msg.write(initProtoField(4, key)) msg.write(initProtoField(4, key))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -345,7 +345,7 @@ proc requestDHTPutValue(key: string, value: openarray[byte],
msg.write(initProtoField(4, key)) msg.write(initProtoField(4, key))
msg.write(initProtoField(5, value)) msg.write(initProtoField(5, value))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -360,7 +360,7 @@ proc requestDHTProvide(cid: Cid, timeout = 0): ProtoBuffer =
msg.write(initProtoField(1, msgid)) msg.write(initProtoField(1, msgid))
msg.write(initProtoField(3, cid.data.buffer)) msg.write(initProtoField(3, cid.data.buffer))
if timeout > 0: if timeout > 0:
msg.write(initProtoField(7, uint(timeout))) msg.write(initProtoField(7, hint64(timeout)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.DHT))) result.write(initProtoField(1, cast[uint](RequestType.DHT)))
result.write(initProtoField(5, msg)) result.write(initProtoField(5, msg))
@ -374,7 +374,7 @@ proc requestCMTagPeer(peer: PeerID, tag: string, weight: int): ProtoBuffer =
msg.write(initProtoField(1, msgid)) msg.write(initProtoField(1, msgid))
msg.write(initProtoField(2, peer)) msg.write(initProtoField(2, peer))
msg.write(initProtoField(3, tag)) msg.write(initProtoField(3, tag))
msg.write(initProtoField(4, weight)) msg.write(initProtoField(4, hint64(weight)))
msg.finish() msg.finish()
result.write(initProtoField(1, cast[uint](RequestType.CONNMANAGER))) result.write(initProtoField(1, cast[uint](RequestType.CONNMANAGER)))
result.write(initProtoField(6, msg)) result.write(initProtoField(6, msg))
@ -949,7 +949,6 @@ proc listPeers*(api: DaemonAPI): Future[seq[PeerInfo]] {.async.} =
try: try:
var pb = await transp.transactMessage(requestListPeers()) var pb = await transp.transactMessage(requestListPeers())
pb.withMessage() do: pb.withMessage() do:
var address = newSeq[byte]()
result = newSeq[PeerInfo]() result = newSeq[PeerInfo]()
var res = pb.enterSubmessage() var res = pb.enterSubmessage()
while res != 0: while res != 0:

2
libp2p/protocols/pubsub/rpc/protobuf.nim
View File

@ -150,7 +150,7 @@ proc encodeSubs*(subs: SubOpts, pb: var ProtoBuffer) {.gcsafe.} =
proc decodeSubs*(pb: var ProtoBuffer): seq[SubOpts] {.gcsafe.} = proc decodeSubs*(pb: var ProtoBuffer): seq[SubOpts] {.gcsafe.} =
while true: while true:
var subOpt: SubOpts var subOpt: SubOpts
var subscr: int var subscr: uint
discard pb.getVarintValue(1, subscr) discard pb.getVarintValue(1, subscr)
subOpt.subscribe = cast[bool](subscr) subOpt.subscribe = cast[bool](subscr)
trace "read subscribe field", subscribe = subOpt.subscribe trace "read subscribe field", subscribe = subOpt.subscribe

200
libp2p/varint.nim
View File

@ -15,7 +15,7 @@
## - LibP2P varint, which is able to encode only 63bits of uint64 number and ## - LibP2P varint, which is able to encode only 63bits of uint64 number and
## maximum size of encoded value is 9 octets (bytes). ## maximum size of encoded value is 9 octets (bytes).
## https://github.com/multiformats/unsigned-varint ## https://github.com/multiformats/unsigned-varint
import bitops import bitops, typetraits
type type
VarintStatus* {.pure.} = enum VarintStatus* {.pure.} = enum
@ -31,21 +31,70 @@ type
LP* = object LP* = object
## Use this type to specify LibP2P varint encoding ## Use this type to specify LibP2P varint encoding
zint32* = distinct int32
zint64* = distinct int64
zint* = distinct int
## Signed integer types which will be encoded using zigzag encoding.
hint32* = distinct int32
hint64* = distinct int64
hint* = distinct int
## Signed integer types which will be encoded using simple cast.
PBSomeUVarint* = uint | uint64 | uint32 PBSomeUVarint* = uint | uint64 | uint32
PBSomeSVarint* = int | int64 | int32 PBSomeSVarint* = hint | hint64 | hint32
PBSomeVarint* = PBSomeUVarint | PBSomeSVarint PBZigVarint* = zint | zint64 | zint32
PBSomeVarint* = PBSomeUVarint | PBSomeSVarint | PBZigVarint
LPSomeUVarint* = uint | uint64 | uint32 | uint16 | uint8 LPSomeUVarint* = uint | uint64 | uint32 | uint16 | uint8
LPSomeVarint* = LPSomeUVarint LPSomeVarint* = LPSomeUVarint
SomeVarint* = PBSomeVarint | LPSomeVarint SomeVarint* = PBSomeVarint | LPSomeVarint
SomeUVarint* = PBSomeUVarint | LPSomeUVarint SomeUVarint* = PBSomeUVarint | LPSomeUVarint
VarintError* = object of CatchableError VarintError* = object of CatchableError
proc vsizeof*(x: SomeVarint): int {.inline.} = proc vsizeof*(x: SomeUVarint): int {.inline.} =
## Returns number of bytes required to encode integer ``x`` as varint. ## Returns number of bytes required to encode integer ``x`` as varint.
if x == cast[type(x)](0): if x == type(x)(0):
result = 1 1
else: else:
result = (fastLog2(x) + 1 + 7 - 1) div 7 (fastLog2(x) + 1 + 7 - 1) div 7
proc vsizeof*(x: PBSomeSVarint): int {.inline.} =
## Returns number of bytes required to encode signed integer ``x``.
##
## Note: This procedure interprets signed integer as ProtoBuffer's
## ``int32`` and ``int64`` integers.
when sizeof(x) == 8:
if int64(x) == 0'i64:
1
else:
(fastLog2(uint64(x)) + 1 + 7 - 1) div 7
else:
if int32(x) == 0'i32:
1
else:
(fastLog2(uint32(x)) + 1 + 7 - 1) div 7
proc vsizeof*(x: PBZigVarint): int {.inline.} =
## Returns number of bytes required to encode signed integer ``x``.
##
## Note: This procedure interprets signed integer as ProtoBuffer's
## ``sint32`` and ``sint64`` integer.
when sizeof(x) == 8:
if int64(x) == 0'i64:
1
else:
if int64(x) < 0'i64:
vsizeof(not(uint64(x) shl 1))
else:
vsizeof(uint64(x) shl 1)
else:
if int32(x) == 0'i32:
1
else:
if int32(x) < 0'i32:
vsizeof(not(uint32(x) shl 1))
else:
vsizeof(uint32(x) shl 1)
proc getUVarint*[T: PB|LP](vtype: typedesc[T], proc getUVarint*[T: PB|LP](vtype: typedesc[T],
pbytes: openarray[byte], pbytes: openarray[byte],
@ -83,16 +132,16 @@ proc getUVarint*[T: PB|LP](vtype: typedesc[T],
var shift = 0'u8 var shift = 0'u8
result = VarintStatus.Incomplete result = VarintStatus.Incomplete
outlen = 0 outlen = 0
outval = cast[type(outval)](0) outval = type(outval)(0)
for i in 0..<len(pbytes): for i in 0..<len(pbytes):
let b = pbytes[i] let b = pbytes[i]
if shift >= MaxBits: if shift >= MaxBits:
result = VarintStatus.Overflow result = VarintStatus.Overflow
outlen = 0 outlen = 0
outval = cast[type(outval)](0) outval = type(outval)(0)
break break
else: else:
outval = outval or (cast[type(outval)](b and 0x7F'u8) shl shift) outval = outval or (type(outval)(b and 0x7F'u8) shl shift)
shift += 7 shift += 7
inc(outlen) inc(outlen)
if (b and 0x80'u8) == 0'u8: if (b and 0x80'u8) == 0'u8:
@ -100,12 +149,12 @@ proc getUVarint*[T: PB|LP](vtype: typedesc[T],
break break
if result == VarintStatus.Incomplete: if result == VarintStatus.Incomplete:
outlen = 0 outlen = 0
outval = cast[type(outval)](0) outval = type(outval)(0)
when vtype is LP: when vtype is LP:
if result == VarintStatus.Success: if result == VarintStatus.Success:
if outlen != vsizeof(outval): if outlen != vsizeof(outval):
outval = cast[type(outval)](0) outval = type(outval)(0)
outlen = 0 outlen = 0
result = VarintStatus.Overlong result = VarintStatus.Overlong
@ -135,16 +184,16 @@ proc putUVarint*[T: PB|LP](vtype: typedesc[T],
when vtype is LP: when vtype is LP:
if sizeof(outval) == 8: if sizeof(outval) == 8:
if (cast[uint64](outval) and 0x8000_0000_0000_0000'u64) != 0'u64: if (uint64(outval) and 0x8000_0000_0000_0000'u64) != 0'u64:
result = Overflow result = Overflow
return return
if value <= cast[type(outval)](0x7F): if value <= type(outval)(0x7F):
buffer[0] = cast[byte](outval and 0xFF) buffer[0] = byte(outval and 0xFF)
inc(k) inc(k)
else: else:
while value != cast[type(outval)](0): while value != type(outval)(0):
buffer[k] = cast[byte]((value and 0x7F) or 0x80) buffer[k] = byte((value and 0x7F) or 0x80)
value = value shr 7 value = value shr 7
inc(k) inc(k)
buffer[k - 1] = buffer[k - 1] and 0x7F'u8 buffer[k - 1] = buffer[k - 1] and 0x7F'u8
@ -158,9 +207,11 @@ proc putUVarint*[T: PB|LP](vtype: typedesc[T],
proc getSVarint*(pbytes: openarray[byte], outsize: var int, proc getSVarint*(pbytes: openarray[byte], outsize: var int,
outval: var PBSomeSVarint): VarintStatus {.inline.} = outval: var PBSomeSVarint): VarintStatus {.inline.} =
## Decode Google ProtoBuf's `signed varint` from buffer ``pbytes`` and store ## Decode signed integer (``int32`` or ``int64``) from buffer ``pbytes``
## it to ``outval``. On success ``outlen`` will be set to number of bytes ## and store it to ``outval``.
## processed while decoding `signed varint`. ##
## On success ``outlen`` will be set to number of bytes processed while
## decoding signed varint.
## ##
## If array ``pbytes`` is empty, ``Incomplete`` error will be returned. ## If array ``pbytes`` is empty, ``Incomplete`` error will be returned.
## ##
@ -181,15 +232,44 @@ proc getSVarint*(pbytes: openarray[byte], outsize: var int,
result = PB.getUVarint(pbytes, outsize, value) result = PB.getUVarint(pbytes, outsize, value)
if result == VarintStatus.Success: if result == VarintStatus.Success:
if (value and cast[type(value)](1)) != cast[type(value)](0): outval = cast[type(outval)](value)
proc getSVarint*(pbytes: openarray[byte], outsize: var int,
outval: var PBZigVarint): VarintStatus {.inline.} =
## Decode Google ProtoBuf's zigzag encoded signed integer (``sint32`` or
## ``sint64`` ) from buffer ``pbytes`` and store it to ``outval``.
##
## On success ``outlen`` will be set to number of bytes processed while
## decoding signed varint.
##
## If array ``pbytes`` is empty, ``Incomplete`` error will be returned.
##
## If there not enough bytes available in array ``pbytes`` to decode `signed
## varint`, ``Incomplete`` error will be returned.
##
## If encoded value can produce integer overflow, ``Overflow`` error will be
## returned.
##
## Note, when decoding 10th byte of 64bit integer only 1 bit from byte will be
## decoded, all other bits will be ignored. When decoding 5th byte of 32bit
## integer only 4 bits from byte will be decoded, all other bits will be
## ignored.
when sizeof(outval) == 8:
var value: uint64
else:
var value: uint32
result = PB.getUVarint(pbytes, outsize, value)
if result == VarintStatus.Success:
if (value and type(value)(1)) != type(value)(0):
outval = cast[type(outval)](not(value shr 1)) outval = cast[type(outval)](not(value shr 1))
else: else:
outval = cast[type(outval)](value shr 1) outval = cast[type(outval)](value shr 1)
proc putSVarint*(pbytes: var openarray[byte], outsize: var int, proc putSVarint*(pbytes: var openarray[byte], outsize: var int,
outval: PBSomeSVarint): VarintStatus {.inline.} = outval: PBZigVarint): VarintStatus {.inline.} =
## Encode Google ProtoBuf's `signed varint` ``outval`` and store it to array ## Encode signed integer ``outval`` using ProtoBuffer's zigzag encoding
## ``pbytes``. ## (``sint32`` or ``sint64``) and store it to array ``pbytes``.
## ##
## On success ``outlen`` will hold number of bytes (octets) used to encode ## On success ``outlen`` will hold number of bytes (octets) used to encode
## unsigned integer ``v``. ## unsigned integer ``v``.
@ -202,18 +282,76 @@ proc putSVarint*(pbytes: var openarray[byte], outsize: var int,
## Maximum encoded length of 32bit integer is 5 octets. ## Maximum encoded length of 32bit integer is 5 octets.
when sizeof(outval) == 8: when sizeof(outval) == 8:
var value: uint64 = var value: uint64 =
if outval < 0: if int64(outval) < 0'i64:
not(cast[uint64](outval) shl 1) not(uint64(outval) shl 1)
else: else:
cast[uint64](outval) shl 1 uint64(outval) shl 1
else: else:
var value: uint32 = var value: uint32 =
if outval < 0: if int32(outval) < 0'i32:
not(cast[uint32](outval) shl 1) not(uint32(outval) shl 1)
else: else:
cast[uint32](outval) shl 1 uint32(outval) shl 1
result = PB.putUVarint(pbytes, outsize, value) result = PB.putUVarint(pbytes, outsize, value)
proc putSVarint*(pbytes: var openarray[byte], outsize: var int,
outval: PBSomeSVarint): VarintStatus {.inline.} =
## Encode signed integer ``outval`` (``int32`` or ``int64``) and store it to
## array ``pbytes``.
##
## On success ``outlen`` will hold number of bytes (octets) used to encode
## unsigned integer ``v``.
##
## If there not enough bytes available in buffer ``pbytes``, ``Incomplete``
## error will be returned and ``outlen`` will be set to number of bytes
## required.
##
## Maximum encoded length of 64bit integer is 10 octets.
## Maximum encoded length of 32bit integer is 5 octets.
when sizeof(outval) == 8:
result = PB.putUVarint(pbytes, outsize, uint64(outval))
else:
result = PB.putUVarint(pbytes, outsize, uint32(outval))
template varintFatal(msg) =
const m = msg
{.fatal: m.}
proc putVarint*[T: PB|LP](vtype: typedesc[T], pbytes: var openarray[byte],
nbytes: var int, value: SomeVarint): VarintStatus {.inline.} =
when vtype is PB:
when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
result = putSVarint(pbytes, nbytes, value)
elif (type(value) is PBSomeUVarint):
result = PB.putUVarint(pbytes, nbytes, value)
else:
varintFatal("Protobuf's varint do not support type [" &
typetraits.name(type(value)) & "]")
elif vtype is LP:
when (type(value) is LPSomeVarint):
result = LP.putUVarint(pbytes, nbytes, value)
else:
varintFatal("LibP2P's varint do not support type [" &
typetraits.name(type(value)) & "]")
proc getVarint*[T: PB|LP](vtype: typedesc[T], pbytes: openarray[byte],
nbytes: var int,
value: var SomeVarint): VarintStatus {.inline.} =
when vtype is PB:
when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
result = getSVarint(pbytes, nbytes, value)
elif (type(value) is PBSomeUVarint):
result = PB.getUVarint(pbytes, nbytes, value)
else:
varintFatal("Protobuf's varint do not support type [" &
typetraits.name(type(value)) & "]")
elif vtype is LP:
when (type(value) is LPSomeVarint):
result = LP.getUVarint(pbytes, nbytes, value)
else:
varintFatal("LibP2P's varint do not support type [" &
typetraits.name(type(value)) & "]")
proc encodeVarint*(vtype: typedesc[PB], proc encodeVarint*(vtype: typedesc[PB],
value: PBSomeVarint): seq[byte] {.inline.} = value: PBSomeVarint): seq[byte] {.inline.} =
## Encode integer to Google ProtoBuf's `signed/unsigned varint` and returns ## Encode integer to Google ProtoBuf's `signed/unsigned varint` and returns
@ -224,7 +362,7 @@ proc encodeVarint*(vtype: typedesc[PB],
result.setLen(5) result.setLen(5)
else: else:
result.setLen(10) result.setLen(10)
when type(value) is PBSomeSVarint: when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
let res = putSVarint(result, outsize, value) let res = putSVarint(result, outsize, value)
else: else:
let res = PB.putUVarint(result, outsize, value) let res = PB.putUVarint(result, outsize, value)

2
libp2p/vbuffer.nim
View File

@ -81,7 +81,7 @@ proc writeSeq*[T: byte|char](vb: var VBuffer, value: openarray[T]) =
## Write array ``value`` to buffer ``vb``, value will be prefixed with ## Write array ``value`` to buffer ``vb``, value will be prefixed with
## varint length of the array. ## varint length of the array.
var length = 0 var length = 0
vb.buffer.setLen(len(vb.buffer) + vsizeof(len(value)) + len(value)) vb.buffer.setLen(len(vb.buffer) + vsizeof(uint(len(value))) + len(value))
let res = LP.putUVarint(toOpenArray(vb.buffer, vb.offset, len(vb.buffer) - 1), let res = LP.putUVarint(toOpenArray(vb.buffer, vb.offset, len(vb.buffer) - 1),
length, uint(len(value))) length, uint(len(value)))
doAssert(res == VarintStatus.Success) doAssert(res == VarintStatus.Success)

14
tests/testinterop.nim
View File

@ -232,6 +232,14 @@ suite "Interop":
proc runTests(): Future[bool] {.async.} = proc runTests(): Future[bool] {.async.} =
var protos = @["/test-stream"] var protos = @["/test-stream"]
var test = "TEST STRING" var test = "TEST STRING"
# We are preparing expect string, which should be prefixed with varint
# length and do not have `\r\n` suffix, because we going to use
# readLine().
var buffer = initVBuffer()
buffer.writeSeq(test & "\r\n")
buffer.finish()
var expect = newString(len(buffer) - 2)
copyMem(addr expect[0], addr buffer.buffer[0], len(expect))
let nativeNode = createNode() let nativeNode = createNode()
let awaiters = await nativeNode.start() let awaiters = await nativeNode.start()
@ -241,8 +249,10 @@ suite "Interop":
var testFuture = newFuture[string]("test.future") var testFuture = newFuture[string]("test.future")
proc daemonHandler(api: DaemonAPI, stream: P2PStream) {.async.} = proc daemonHandler(api: DaemonAPI, stream: P2PStream) {.async.} =
# We should perform `readLp()` instead of `readLine()`. `readLine()`
# here reads actually length prefixed string.
var line = await stream.transp.readLine() var line = await stream.transp.readLine()
check line == test check line == expect
testFuture.complete(line) testFuture.complete(line)
await daemonNode.addHandler(protos, daemonHandler) await daemonNode.addHandler(protos, daemonHandler)
@ -250,7 +260,7 @@ suite "Interop":
daemonPeer.addresses), daemonPeer.addresses),
protos[0]) protos[0])
await conn.writeLp(test & "\r\n") await conn.writeLp(test & "\r\n")
result = test == (await wait(testFuture, 10.secs)) result = expect == (await wait(testFuture, 10.secs))
await nativeNode.stop() await nativeNode.stop()
await allFutures(awaiters) await allFutures(awaiters)
await daemonNode.close() await daemonNode.close()

204
tests/testvarint.nim
View File

@ -16,6 +16,84 @@ const PBedgeValues = [
(1'u64 shl 63), 0xFFFF_FFFF_FFFF_FFFF'u64 (1'u64 shl 63), 0xFFFF_FFFF_FFFF_FFFF'u64
] ]
const PBPositiveSignedEdgeValues = [
0'u64, 0x3F'u64,
0x40'u64, 0x1FFF'u64,
0x2000'u64, 0xFFFFF'u64,
0x100000'u64, 0x7FFFFFF'u64,
0x8000000'u64, 0x3FFFFFFFF'u64,
0x400000000'u64, 0x1FFFFFFFFFF'u64,
0x20000000000'u64, 0xFFFFFFFFFFFF'u64,
0x1000000000000'u64, 0x7FFFFFFFFFFFFF'u64,
0x80000000000000'u64, 0x3FFFFFFFFFFFFFFF'u64,
0x4000000000000000'u64, 0x7FFFFFFFFFFFFFFF'u64
]
const PBNegativeSignedEdgeValues = [
0x0000000000000000'u64, 0xFFFFFFFFFFFFFFC0'u64,
0xFFFFFFFFFFFFFFBF'u64, 0xFFFFFFFFFFFFE000'u64,
0xFFFFFFFFFFFFDFFF'u64, 0xFFFFFFFFFFF00000'u64,
0xFFFFFFFFFFEFFFFF'u64, 0xFFFFFFFFF8000000'u64,
0xFFFFFFFFF7FFFFFF'u64, 0xFFFFFFFC00000000'u64,
0xFFFFFFFBFFFFFFFF'u64, 0xFFFFFE0000000000'u64,
0xFFFFFDFFFFFFFFFF'u64, 0xFFFF000000000000'u64,
0xFFFEFFFFFFFFFFFF'u64, 0xFF80000000000000'u64,
0xFF7FFFFFFFFFFFFF'u64, 0xC000000000000000'u64,
0xBFFFFFFFFFFFFFFF'u64, 0x8000000000000000'u64
]
const PBPositiveSignedZigZagEdgeExpects = [
"00", "7E",
"8001", "FE7F",
"808001", "FEFF7F",
"80808001", "FEFFFF7F",
"8080808001", "FEFFFFFF7F",
"808080808001", "FEFFFFFFFF7F",
"80808080808001", "FEFFFFFFFFFF7F",
"8080808080808001", "FEFFFFFFFFFFFF7F",
"808080808080808001", "FEFFFFFFFFFFFFFF7F",
"80808080808080808001", "FEFFFFFFFFFFFFFFFF01"
]
const PBNegativeSignedZigZagEdgeExpects = [
"00", "7F",
"8101", "FF7F",
"818001", "FFFF7F",
"81808001", "FFFFFF7F",
"8180808001", "FFFFFFFF7F",
"818080808001", "FFFFFFFFFF7F",
"81808080808001", "FFFFFFFFFFFF7F",
"8180808080808001", "FFFFFFFFFFFFFF7F",
"818080808080808001", "FFFFFFFFFFFFFFFF7F",
"81808080808080808001", "FFFFFFFFFFFFFFFFFF01",
]
const PBPositiveSignedEdgeExpects = [
"00", "3F",
"40", "FF3F",
"8040", "FFFF3F",
"808040", "FFFFFF3F",
"80808040", "FFFFFFFF3F",
"8080808040", "FFFFFFFFFF3F",
"808080808040", "FFFFFFFFFFFF3F",
"80808080808040", "FFFFFFFFFFFFFF3F",
"8080808080808040", "FFFFFFFFFFFFFFFF3F",
"808080808080808040", "FFFFFFFFFFFFFFFF7F"
]
const PBNegativeSignedEdgeExpects = [
"00", "C0FFFFFFFFFFFFFFFF01",
"BFFFFFFFFFFFFFFFFF01", "80C0FFFFFFFFFFFFFF01",
"FFBFFFFFFFFFFFFFFF01", "8080C0FFFFFFFFFFFF01",
"FFFFBFFFFFFFFFFFFF01", "808080C0FFFFFFFFFF01",
"FFFFFFBFFFFFFFFFFF01", "80808080C0FFFFFFFF01",
"FFFFFFFFBFFFFFFFFF01", "8080808080C0FFFFFF01",
"FFFFFFFFFFBFFFFFFF01", "808080808080C0FFFF01",
"FFFFFFFFFFFFBFFFFF01", "80808080808080C0FF01",
"FFFFFFFFFFFFFFBFFF01", "8080808080808080C001",
"FFFFFFFFFFFFFFFFBF01", "80808080808080808001"
]
const PBedgeExpects = [ const PBedgeExpects = [
"00", "7F", "00", "7F",
"8001", "FF7F", "8001", "FF7F",
@ -33,6 +111,22 @@ const PBedgeSizes = [
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10
] ]
const PBEdgeSignedPositiveZigZagSizes = [
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10
]
const PBEdgeSignedNegativeZigZagSizes = [
1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10
]
const PBEdgeSignedPositiveSizes = [
1, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 9
]
const PBEdgeSignedNegativeSizes = [
1, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10
]
const LPedgeValues = [ const LPedgeValues = [
0'u64, (1'u64 shl 7) - 1'u64, 0'u64, (1'u64 shl 7) - 1'u64,
(1'u64 shl 7), (1'u64 shl 14) - 1'u64, (1'u64 shl 7), (1'u64 shl 14) - 1'u64,
@ -85,21 +179,72 @@ proc toHex*(a: openarray[byte], lowercase: bool = false): string =
suite "Variable integer test suite": suite "Variable integer test suite":
test "vsizeof() edge cases test": test "vsizeof() edge cases test":
for i in 0..<len(PBedgeValues): for i in 0 ..< len(PBedgeValues):
check vsizeof(PBedgeValues[i]) == PBedgeSizes[i] check vsizeof(PBedgeValues[i]) == PBedgeSizes[i]
for i in 0 ..< len(PBPositiveSignedEdgeValues):
check:
vsizeof(hint64(PBPositiveSignedEdgeValues[i])) ==
PBEdgeSignedPositiveSizes[i]
vsizeof(zint64(PBPositiveSignedEdgeValues[i])) ==
PBEdgeSignedPositiveZigZagSizes[i]
for i in 0 ..< len(PBNegativeSignedEdgeValues):
check:
vsizeof(hint64(PBNegativeSignedEdgeValues[i])) ==
PBEdgeSignedNegativeSizes[i]
vsizeof(zint64(PBNegativeSignedEdgeValues[i])) ==
PBEdgeSignedNegativeZigZagSizes[i]
test "[ProtoBuf] Success edge cases test": test "[ProtoBuf] Success edge cases test":
var buffer = newSeq[byte]() var buffer = newSeq[byte]()
var length = 0 var length = 0
var value = 0'u64 var uvalue = 0'u64
for i in 0..<len(PBedgeValues): var ivalue = hint64(0)
var svalue = zint64(0)
for i in 0 ..< len(PBedgeValues):
buffer.setLen(PBedgeSizes[i]) buffer.setLen(PBedgeSizes[i])
check: check:
PB.putUVarint(buffer, length, PBedgeValues[i]) == VarintStatus.Success PB.putUVarint(buffer, length, PBedgeValues[i]) == VarintStatus.Success
PB.getUVarint(buffer, length, value) == VarintStatus.Success PB.getUVarint(buffer, length, uvalue) == VarintStatus.Success
value == PBedgeValues[i] uvalue == PBedgeValues[i]
toHex(buffer) == PBedgeExpects[i] toHex(buffer) == PBedgeExpects[i]
for i in 0 ..< len(PBPositiveSignedEdgeValues):
buffer.setLen(PBEdgeSignedPositiveSizes[i])
check:
putSVarint(buffer, length,
hint64(PBPositiveSignedEdgeValues[i])) == VarintStatus.Success
getSVarint(buffer, length, ivalue) == VarintStatus.Success
int64(ivalue) == int64(PBPositiveSignedEdgeValues[i])
toHex(buffer) == PBPositiveSignedEdgeExpects[i]
buffer.setLen(PBEdgeSignedPositiveZigZagSizes[i])
check:
putSVarint(buffer, length,
zint64(PBPositiveSignedEdgeValues[i])) == VarintStatus.Success
getSVarint(buffer, length, svalue) == VarintStatus.Success
int64(svalue) == int64(PBPositiveSignedEdgeValues[i])
toHex(buffer) == PBPositiveSignedZigZagEdgeExpects[i]
for i in 0 ..< len(PBNegativeSignedEdgeValues):
buffer.setLen(PBEdgeSignedNegativeSizes[i])
check:
putSVarint(buffer, length,
hint64(PBNegativeSignedEdgeValues[i])) == VarintStatus.Success
getSVarint(buffer, length, ivalue) == VarintStatus.Success
int64(ivalue) == int64(PBNegativeSignedEdgeValues[i])
toHex(buffer) == PBNegativeSignedEdgeExpects[i]
buffer.setLen(PBEdgeSignedNegativeZigZagSizes[i])
check:
putSVarint(buffer, length,
zint64(PBNegativeSignedEdgeValues[i])) == VarintStatus.Success
getSVarint(buffer, length, svalue) == VarintStatus.Success
int64(svalue) == int64(PBNegativeSignedEdgeValues[i])
toHex(buffer) == PBNegativeSignedZigZagEdgeExpects[i]
test "[ProtoBuf] Buffer Overrun edge cases test": test "[ProtoBuf] Buffer Overrun edge cases test":
var buffer = newSeq[byte]() var buffer = newSeq[byte]()
var length = 0 var length = 0
@ -147,6 +292,23 @@ suite "Variable integer test suite":
check: check:
PB.getUVarint(buffer, length, value) == VarintStatus.Overflow PB.getUVarint(buffer, length, value) == VarintStatus.Overflow
test "[ProtoBuf] Test vectors":
# The test vectors which was obtained at:
# https://github.com/dermesser/integer-encoding-rs/blob/master/src/varint_tests.rs
# https://github.com/That3Percent/zigzag/blob/master/src/lib.rs
check:
PB.encodeVarint(0'u64) == @[0x00'u8]
PB.encodeVarint(0'u32) == @[0x00'u8]
PB.encodeVarint(hint64(0)) == @[0x00'u8]
PB.encodeVarint(hint32(0)) == @[0x00'u8]
PB.encodeVarint(zint64(0)) == @[0x00'u8]
PB.encodeVarint(zint32(0)) == @[0x00'u8]
PB.encodeVarint(zint32(-1)) == PB.encodeVarint(1'u32)
PB.encodeVarint(zint64(150)) == PB.encodeVarint(300'u32)
PB.encodeVarint(zint64(-150)) == PB.encodeVarint(299'u32)
PB.encodeVarint(zint32(-2147483648)) == PB.encodeVarint(4294967295'u64)
PB.encodeVarint(zint32(2147483647)) == PB.encodeVarint(4294967294'u64)
test "[LibP2P] Success edge cases test": test "[LibP2P] Success edge cases test":
var buffer = newSeq[byte]() var buffer = newSeq[byte]()
var length = 0 var length = 0
@ -267,3 +429,35 @@ suite "Variable integer test suite":
LP.getUVarint(@[0x80'u8, 0x00'u8], length, value) == VarintStatus.Overlong LP.getUVarint(@[0x80'u8, 0x00'u8], length, value) == VarintStatus.Overlong
length == 0 length == 0
value == 0 value == 0
test "getVarint/putVarint tests":
proc `==`(a, b: zint32|hint32): bool =
int32(a) == int32(b)
proc `==`(a, b: zint64|hint64): bool =
int64(a) == int64(b)
template varintTest(ttype, vtype, value, expect: untyped) =
var ovalue: vtype
var buffer = newSeq[byte](10)
var length = 0
check ttype.putVarint(buffer, length, value) == VarintStatus.Success
buffer.setLen(length)
check:
toHex(buffer) == expect
ttype.getVarint(buffer, length, ovalue) == VarintStatus.Success
ovalue == value
varintTest(PB, uint64, high(uint64), "FFFFFFFFFFFFFFFFFF01")
varintTest(PB, uint32, high(uint32), "FFFFFFFF0F")
varintTest(PB, zint64, zint64(high(int64)), "FEFFFFFFFFFFFFFFFF01")
varintTest(PB, zint32, zint32(high(int32)), "FEFFFFFF0F")
varintTest(PB, zint64, zint64(low(int64)), "FFFFFFFFFFFFFFFFFF01")
varintTest(PB, zint32, zint32(low(int32)), "FFFFFFFF0F")
varintTest(PB, hint64, hint64(high(int64)), "FFFFFFFFFFFFFFFF7F")
varintTest(PB, hint32, hint32(high(int32)), "FFFFFFFF07")
varintTest(PB, hint64, hint64(low(int64)), "80808080808080808001")
varintTest(PB, hint32, hint32(low(int32)), "8080808008")
varintTest(LP, uint64, uint64(high(int64)), "FFFFFFFFFFFFFFFF7F")
varintTest(LP, uint32, uint32(high(uint32)), "FFFFFFFF0F")
varintTest(LP, uint16, uint16(high(uint16)), "FFFF03")
varintTest(LP, uint8, uint8(high(uint8)), "FF01")