Extract the varints code being used in the latest LibP2P back-end
Originally, this code was taken from https://github.com/status-im/nim-libp2p/blob/master/libp2p/varint.nim The API was refactored to make it easier to use in the networking back-ends where the data is being consumed byte by byte.
This commit is contained in:
Zahary Karadjov
parent
9ac177db1f
commit
6be00f7c4b
|
|
@ -0,0 +1,203 @@
|
|||
## This module implements Variable Integer `VARINT`.
|
||||
|
||||
import
|
||||
bitops2
|
||||
|
||||
type
|
||||
VarintFlavour* = enum
|
||||
ProtoBuf
|
||||
LibP2P
|
||||
|
||||
VarintState* {.pure.} = enum
|
||||
Incomplete,
|
||||
Done,
|
||||
Overflow
|
||||
|
||||
VarintParser*[IntType; flavour: static VarintFlavour] = object
|
||||
## This stateful object can be used to parse varints.
|
||||
##
|
||||
## Type parameters:
|
||||
##
|
||||
## * `IntType` - The output type the parser will try to read
|
||||
## * `flavour` - The type of varint encoding.
|
||||
##
|
||||
## The following encodings are supported:
|
||||
##
|
||||
## * `ProtoBuf`
|
||||
##
|
||||
## The encoding used by Google ProtoBuf.
|
||||
## It's able to encode a full uint64 number and the maximum
|
||||
## encoded size is 10 octets (bytes).
|
||||
##
|
||||
## When decoding 10th byte of Google Protobuf's 64bit integer
|
||||
## only 1 bit from byte will be decoded, all other bits will
|
||||
## be ignored. When decoding 5th byte of 32bit integer only
|
||||
## 4 bits from byte will be decoded, all other bits will be
|
||||
## ignored.
|
||||
##
|
||||
## * `LibP2P`
|
||||
##
|
||||
## Encoding used by the LibP2P project.
|
||||
## It ca encode only lower 63 bits of a uint64 number with a
|
||||
## maximum size for the encoded value of 9 octets (bytes).
|
||||
##
|
||||
## When decoding 5th byte of 32bit integer only 4 bits from
|
||||
## byte will be decoded, all other bits will be ignored.
|
||||
##
|
||||
## Usage protocol:
|
||||
##
|
||||
## This object is initialized with the default zero-initialization.
|
||||
## Proceed to calling `feedByte` one or multiple times and then obtain
|
||||
## the result with `getResult`.
|
||||
##
|
||||
shift: uint8
|
||||
when IntType is int64|uint64:
|
||||
res: uint64
|
||||
else:
|
||||
res: uint32
|
||||
when defined(debug):
|
||||
state: VarintState
|
||||
|
||||
func maxBits(T: type VarintParser): uint8 {.compileTime.} =
|
||||
when T.flavour == ProtoBuf:
|
||||
uint8(sizeof(T.IntType) * 8)
|
||||
elif sizeof(T.IntType) == 8:
|
||||
uint8(63)
|
||||
else:
|
||||
uint(sizeof(T.IntType) * 8)
|
||||
|
||||
func feedByte*(p: var VarintParser, b: byte): VarintState =
|
||||
## Supplies the next input byte to the varint parser.
|
||||
## The return value is one of the following:
|
||||
##
|
||||
## * `Incomplete`
|
||||
## More input bytes must be supplied.
|
||||
##
|
||||
## * `Done`
|
||||
## The varint has been read to completion.
|
||||
## Use `parser.getResult` to obtain it.
|
||||
##
|
||||
## * `Overflow`
|
||||
## The maximum number of bits in the parser output value
|
||||
## has been exceed. The supplied input can be considered invalid.
|
||||
##
|
||||
const maxShift = maxBits type(p)
|
||||
|
||||
if p.shift >= maxShift:
|
||||
return Overflow
|
||||
|
||||
p.res = p.res or (uint64(b and 0x7F'u8) shl p.shift)
|
||||
p.shift += 7
|
||||
|
||||
if (b and 0x80'u8) == 0'u8:
|
||||
when defined(debug): p.state = Done
|
||||
Done
|
||||
else:
|
||||
Incomplete
|
||||
|
||||
func getResult*[IntType, F](p: VarintParser[IntType, F]): IntType {.inline.} =
|
||||
## Returns the final result of the varint parsing.
|
||||
## This function must be called after a previous call to `parser.feedByte`
|
||||
## has returned the state `Done`. The result of calling the function at
|
||||
## any other time is undefined.
|
||||
when defined(debug):
|
||||
doAssert p.state == Done
|
||||
|
||||
when result is SomeSignedInt:
|
||||
type UIntType = type(p.res)
|
||||
|
||||
if p.res and UIntType(1) != UIntType(0):
|
||||
cast[p.IntType](not (p.res shr 1))
|
||||
else:
|
||||
cast[p.IntType](p.res shr 1)
|
||||
else:
|
||||
p.res
|
||||
|
||||
func readVarint*(input: openarray[byte],
|
||||
outVal: var SomeInteger,
|
||||
flavour: static VarintFlavour = ProtoBuf): int =
|
||||
## Reads a varint from a buffer and stores it in `outVal`.
|
||||
## The return value indicates the number of bytes read.
|
||||
## If the buffer doesn't hold a valid varint value, the
|
||||
## function will return 0.
|
||||
var
|
||||
parser: VarintParser[type(outVal), flavour]
|
||||
pos = 0
|
||||
|
||||
while pos < input.len:
|
||||
case parser.feedByte(input[pos])
|
||||
of Incomplete:
|
||||
inc pos
|
||||
of Done:
|
||||
outVal = parser.getResult
|
||||
return pos + 1
|
||||
of Overflow:
|
||||
return 0
|
||||
|
||||
func readVarint*[Stream](input: var Stream,
|
||||
T: type SomeInteger,
|
||||
flavour: static VarintFlavour = ProtoBuf): T =
|
||||
## Reads a varint from a stream (e.g. fastreams.InputStream) and returns it.
|
||||
##
|
||||
## The following exceptions may be raised:
|
||||
##
|
||||
## * `EOFError`
|
||||
## The end of the stream was reached before the varint
|
||||
## was completely read.
|
||||
##
|
||||
## * `ValueError`
|
||||
## The stream contained an invalid varint value.
|
||||
var
|
||||
parser: VarintParser[type(outVal), flavour]
|
||||
pos = input.pos
|
||||
|
||||
while not input.eof:
|
||||
case parser.feedByte(input.read)
|
||||
of Done:
|
||||
return parser.getResult
|
||||
of Overflow:
|
||||
raise newException(ValueError, "Failed to read a varint")
|
||||
of Incomplete:
|
||||
continue
|
||||
|
||||
raise newException(EOFError, "Failed to read a varint")
|
||||
|
||||
func appendVarintImpl[Stream](s: var Stream, x: SomeUnsignedInt) {.inline.} =
|
||||
mixin append
|
||||
type UInt = type(x)
|
||||
|
||||
if x <= 0x7F:
|
||||
s.append byte(x and 0xFF)
|
||||
else:
|
||||
var x = x
|
||||
while true:
|
||||
var nextByte = byte((x and 0x7F) or 0x80)
|
||||
x = x shr 7
|
||||
if x == 0:
|
||||
nextByte = nextByte and 0x7F
|
||||
s.append nextByte
|
||||
return
|
||||
else:
|
||||
s.append nextByte
|
||||
|
||||
func appendVarint*[Stream](s: var Stream, x: SomeInteger,
|
||||
flavour: static VarintFlavour = ProtoBuf) {.inline.} =
|
||||
## Writes a varint to a stream (e.g. faststreams.OutputStream)
|
||||
when x is SomeSignedInt:
|
||||
type UInt = (when sizeof(x) == 8: uint64
|
||||
elif sizeof(x) == 4: uint32
|
||||
else: uint16)
|
||||
|
||||
let x = if x < 0: not(cast[UInt](x) shl 1)
|
||||
else: cast[UInt](x) shl 1
|
||||
|
||||
when flavour == LibP2P and sizeof(x) == 8:
|
||||
doAssert(x.getBitBE(0) == false)
|
||||
|
||||
s.appendVarintImpl x
|
||||
|
||||
func vsizeof*(x: SomeInteger): int {.inline.} =
|
||||
## Returns number of bytes required to encode integer ``x`` as varint.
|
||||
if x == 0: 1
|
||||
else: (log2trunc(x) + 1 + 7 - 1) div 7
|
||||
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
import
|
||||
unittest, random,
|
||||
../stew/[varints, byteutils]
|
||||
|
||||
const edgeValues = {
|
||||
0'u64 : "00",
|
||||
(1'u64 shl 7) - 1'u64 : "7f",
|
||||
(1'u64 shl 7) : "8001",
|
||||
(1'u64 shl 14) - 1'u64 : "ff7f",
|
||||
(1'u64 shl 14) : "808001",
|
||||
(1'u64 shl 21) - 1'u64 : "ffff7f",
|
||||
(1'u64 shl 21) : "80808001",
|
||||
(1'u64 shl 28) - 1'u64 : "ffffff7f",
|
||||
(1'u64 shl 28) : "8080808001",
|
||||
(1'u64 shl 35) - 1'u64 : "ffffffff7f",
|
||||
(1'u64 shl 35) : "808080808001",
|
||||
(1'u64 shl 42) - 1'u64 : "ffffffffff7f",
|
||||
(1'u64 shl 42) : "80808080808001",
|
||||
(1'u64 shl 49) - 1'u64 : "ffffffffffff7f",
|
||||
(1'u64 shl 49) : "8080808080808001",
|
||||
(1'u64 shl 56) - 1'u64 : "ffffffffffffff7f",
|
||||
(1'u64 shl 56) : "808080808080808001",
|
||||
(1'u64 shl 63) - 1'u64 : "ffffffffffffffff7f",
|
||||
uint64(1'u64 shl 63) : "80808080808080808001",
|
||||
0xFFFF_FFFF_FFFF_FFFF'u64 : "ffffffffffffffffff01"
|
||||
}
|
||||
|
||||
type
|
||||
PseudoStream = object
|
||||
bytes: array[12, byte]
|
||||
bytesWritten: int
|
||||
|
||||
func append(s: var PseudoStream, b: byte) =
|
||||
s.bytes[s.bytesWritten] = b
|
||||
inc s.bytesWritten
|
||||
|
||||
template writtenData(s: PseudoStream): auto =
|
||||
s.bytes.toOpenArray(0, s.bytesWritten - 1)
|
||||
|
||||
suite "varints":
|
||||
template roundtipTest(val) =
|
||||
var s {.inject.}: PseudoStream
|
||||
s.appendVarint val
|
||||
|
||||
var roundtripVal: uint64
|
||||
let bytesRead = readVarint(s.bytes, roundtripVal)
|
||||
|
||||
check:
|
||||
val == roundtripVal
|
||||
bytesRead == s.bytesWritten
|
||||
bytesRead == vsizeof(val)
|
||||
|
||||
test "[ProtoBuf] Success edge cases test":
|
||||
for pair in edgeValues:
|
||||
let (val, hex) = pair
|
||||
roundtipTest val
|
||||
check:
|
||||
s.bytesWritten == hex.len div 2
|
||||
toHex(s.writtenData) == hex
|
||||
|
||||
test "[ProtoBuf] random values":
|
||||
for i in 0..10000:
|
||||
let val = rand(0'u64 .. 0xFFFF_FFFF_FFFF_FFFE'u64)
|
||||
roundtipTest val
|
||||
|
||||
# TODO Migrate the rest of the LibP2P test cases
|
||||
Loading…
Reference in New Issue