nim-libp2p/libp2p/varint.nim

# Nim-Libp2p
# Copyright (c) 2022 Status Research & Development GmbH
# Licensed under either of
#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.

## This module implements Variable Integer `VARINT`.
## This module supports two variants of variable integer
## - Google ProtoBuf varint, which is able to encode full uint64 number and
##   maximum size of encoded value is 10 octets (bytes).
##   https://developers.google.com/protocol-buffers/docs/encoding#varints
## - LibP2P varint, which is able to encode only 63bits of uint64 number and
##   maximum size of encoded value is 9 octets (bytes).
##   https://github.com/multiformats/unsigned-varint

when (NimMajor, NimMinor) < (1, 4):
  {.push raises: [Defect].}
else:
  {.push raises: [].}

import stew/[byteutils, leb128, results]
export leb128, results

type
  VarintError* {.pure.} = enum
    Error,
    Overflow,
    Incomplete,
    Overlong,
    Overrun

  VarintResult*[T] = Result[T, VarintError]

  PB* = object
    ## Use this type to specify Google ProtoBuf's varint encoding
  LP* = object
    ## Use this type to specify LibP2P varint encoding

  zint32* = distinct int32
  zint64* = distinct int64
  zint* = distinct int
    ## Signed integer types which will be encoded using zigzag encoding.

  hint32* = distinct int32
  hint64* = distinct int64
  hint* = distinct int
    ## Signed integer types which will be encoded using simple cast.

  PBSomeUVarint* = uint | uint64 | uint32
  PBSomeSVarint* = hint | hint64 | hint32
  PBZigVarint* = zint | zint64 | zint32
  PBSomeVarint* = PBSomeUVarint | PBSomeSVarint | PBZigVarint
  LPSomeUVarint* = uint | uint64 | uint32 | uint16 | uint8
  LPSomeVarint* = LPSomeUVarint
  SomeVarint* = PBSomeVarint | LPSomeVarint
  SomeUVarint* = PBSomeUVarint | LPSomeUVarint

template toUleb[T: uint64|uint32|uint16|uint8|uint](x: T): T = x

func toUleb(x: zint64): uint64 =
  let v = cast[uint64](x)
  (v shl 1) xor (0 - (v shr 63))

func toUleb(x: zint32): uint32 =
  let v = cast[uint32](x)
  (v shl 1) xor (0 - (v shr 31))

template toUleb(x: hint64): uint64 = cast[uint64](x)
template toUleb(x: hint32): uint32 = cast[uint32](x)

template toUleb(x: zint): uint64 =
  when sizeof(x) == sizeof(zint64):
    uint(toUleb(zint64(x)))
  else:
    uint(toUleb(zint32(x)))

template toUleb(x: hint): uint =
  when sizeof(x) == sizeof(hint64):
    uint(toUleb(hint64(x)))
  else:
    uint(toUleb(hint32(x)))

template fromUleb(x: uint64, T: type uint64): T = x
template fromUleb(x: uint32, T: type uint32): T = x

template fromUleb(x: uint64, T: type zint64): T =
  cast[T]((x shr 1) xor (0 - (x and 1)))

template fromUleb(x: uint32, T: type zint32): T =
  cast[T]((x shr 1) xor (0 - (x and 1)))

template fromUleb(x: uint64, T: type hint64): T =
  cast[T](x)

template fromUleb(x: uint32, T: type hint32): T =
  cast[T](x)

proc vsizeof*(x: SomeVarint): int {.inline.} =
  ## Returns number of bytes required to encode integer ``x`` as varint.
  Leb128.len(toUleb(x))

proc getUVarint*[T: PB|LP](vtype: typedesc[T],
                           pbytes: openArray[byte],
                           outlen: var int,
                           outval: var SomeUVarint): VarintResult[void] =
  ## Decode `unsigned varint` from buffer ``pbytes`` and store it to ``outval``.
  ## On success ``outlen`` will be set to number of bytes processed while
  ## decoding `unsigned varint`.
  ##
  ## If array ``pbytes`` is empty, ``Incomplete`` error will be returned.
  ##
  ## If there not enough bytes available in array ``pbytes`` to decode `unsigned
  ## varint`, ``Incomplete`` error will be returned.
  ##
  ## If encoded value can produce integer overflow, ``Overflow`` error will be
  ## returned.
  ##
  ## Google ProtoBuf
  ## When decoding 10th byte of Google Protobuf's 64bit integer only 1 bit from
  ## byte will be decoded, all other bits will be ignored. When decoding 5th
  ## byte of 32bit integer only 4 bits from byte will be decoded, all other bits
  ## will be ignored.
  ##
  ## LibP2P
  ## When decoding 5th byte of 32bit integer only 4 bits from byte will be
  ## decoded, all other bits will be ignored.
  outlen = 0
  outval = type(outval)(0)

  let parsed = type(outval).fromBytes(pbytes, Leb128)

  if parsed.len == 0:
    return err(VarintError.Incomplete)
  if parsed.len < 0:
    return err(VarintError.Overflow)

  when vtype is LP and sizeof(outval) == 8:
    if parsed.val >= 0x8000_0000_0000_0000'u64:
      return err(VarintError.Overflow)

  if vsizeof(parsed.val) != parsed.len:
    return err(VarintError.Overlong)

  (outval, outlen) = parsed

  ok()

proc putUVarint*[T: PB|LP](vtype: typedesc[T],
                           pbytes: var openArray[byte],
                           outlen: var int,
                           outval: SomeUVarint): VarintResult[void] =
  ## Encode `unsigned varint` ``outval`` and store it to array ``pbytes``.
  ##
  ## On success ``outlen`` will hold number of bytes (octets) used to encode
  ## unsigned integer ``v``.
  ##
  ## If there not enough bytes available in buffer ``pbytes``, ``Incomplete``
  ## error will be returned and ``outlen`` will be set to number of bytes
  ## required.
  ##
  ## Google ProtoBuf
  ## Maximum encoded length of 64bit integer is 10 octets.
  ## Maximum encoded length of 32bit integer is 5 octets.
  ##
  ## LibP2P
  ## Maximum encoded length of 63bit integer is 9 octets.
  ## Maximum encoded length of 32bit integer is 5 octets.
  when vtype is LP and sizeof(outval) == 8:
    if (uint64(outval) and 0x8000_0000_0000_0000'u64) != 0'u64:
      return err(VarintError.Overflow)

  let bytes = toBytes(outval, Leb128)
  outlen = len(bytes)
  if len(pbytes) >= outlen:
    pbytes[0..<outlen] = bytes.toOpenArray()
    ok()
  else:
    err(VarintError.Overrun)

proc getSVarint*(pbytes: openArray[byte], outsize: var int,
                 outval: var (PBZigVarint | PBSomeSVarint)): VarintResult[void] {.inline.} =
  ## Decode signed integer (``int32`` or ``int64``) from buffer ``pbytes``
  ## and store it to ``outval``.
  ##
  ## On success ``outlen`` will be set to number of bytes processed while
  ## decoding signed varint.
  ##
  ## If array ``pbytes`` is empty, ``Incomplete`` error will be returned.
  ##
  ## If there not enough bytes available in array ``pbytes`` to decode `signed
  ## varint`, ``Incomplete`` error will be returned.
  ##
  ## If encoded value can produce integer overflow, ``Overflow`` error will be
  ## returned.
  ##
  ## Note, when decoding 10th byte of 64bit integer only 1 bit from byte will be
  ## decoded, all other bits will be ignored. When decoding 5th byte of 32bit
  ## integer only 4 bits from byte will be decoded, all other bits will be
  ## ignored.
  when sizeof(outval) == 8:
    var value: uint64
  else:
    var value: uint32

  let res = PB.getUVarint(pbytes, outsize, value)
  if res.isOk():
    outval = fromUleb(value, type(outval))
  res

proc putSVarint*(pbytes: var openArray[byte], outsize: var int,
                 outval: (PBZigVarint | PBSomeSVarint)): VarintResult[void] {.inline.} =
  ## Encode signed integer ``outval`` using ProtoBuffer's zigzag encoding
  ## (``sint32`` or ``sint64``) and store it to array ``pbytes``.
  ##
  ## On success ``outlen`` will hold number of bytes (octets) used to encode
  ## unsigned integer ``v``.
  ##
  ## If there not enough bytes available in buffer ``pbytes``, ``Incomplete``
  ## error will be returned and ``outlen`` will be set to number of bytes
  ## required.
  ##
  ## Maximum encoded length of 64bit integer is 10 octets.
  ## Maximum encoded length of 32bit integer is 5 octets.
  PB.putUVarint(pbytes, outsize, toUleb(outval))

template varintFatal(msg) =
  const m = msg
  {.fatal: m.}

proc putVarint*[T: PB|LP](vtype: typedesc[T], pbytes: var openArray[byte],
                nbytes: var int, value: SomeVarint): VarintResult[void] {.inline.} =
  when vtype is PB:
    when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
      putSVarint(pbytes, nbytes, value)
    elif (type(value) is PBSomeUVarint):
      PB.putUVarint(pbytes, nbytes, value)
    else:
      varintFatal("Protobuf's varint do not support type [" &
                  typetraits.name(type(value)) & "]")
  elif vtype is LP:
    when (type(value) is LPSomeVarint):
      LP.putUVarint(pbytes, nbytes, value)
    else:
      varintFatal("LibP2P's varint do not support type [" &
                   typetraits.name(type(value)) & "]")

proc getVarint*[T: PB|LP](vtype: typedesc[T], pbytes: openArray[byte],
                          nbytes: var int,
                          value: var SomeVarint): VarintResult[void] {.inline.} =
  when vtype is PB:
    when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
      getSVarint(pbytes, nbytes, value)
    elif (type(value) is PBSomeUVarint):
      PB.getUVarint(pbytes, nbytes, value)
    else:
      varintFatal("Protobuf's varint do not support type [" &
                  typetraits.name(type(value)) & "]")
  elif vtype is LP:
    when (type(value) is LPSomeVarint):
      LP.getUVarint(pbytes, nbytes, value)
    else:
      varintFatal("LibP2P's varint do not support type [" &
                   typetraits.name(type(value)) & "]")

template toBytes*(vtype: typedesc[PB], value: PBSomeVarint): auto =
  toBytes(toUleb(value), Leb128)

proc encodeVarint*(vtype: typedesc[PB],
                   value: PBSomeVarint): VarintResult[seq[byte]] {.inline.} =
  ## Encode integer to Google ProtoBuf's `signed/unsigned varint` and returns
  ## sequence of bytes as buffer.
  var outsize = 0
  var buffer = newSeqOfCap[byte](10)
  when sizeof(value) == 4:
    buffer.setLen(5)
  else:
    buffer.setLen(10)
  when (type(value) is PBSomeSVarint) or (type(value) is PBZigVarint):
    let res = putSVarint(buffer, outsize, value)
  else:
    let res = PB.putUVarint(buffer, outsize, value)
  if res.isOk():
    buffer.setLen(outsize)
    ok(buffer)
  else:
    err(res.error())

proc encodeVarint*(vtype: typedesc[LP],
                   value: LPSomeVarint): VarintResult[seq[byte]] {.inline.} =
  ## Encode integer to LibP2P `unsigned varint` and returns sequence of bytes
  ## as buffer.
  var outsize = 0
  var buffer = newSeqOfCap[byte](9)
  when sizeof(value) == 1:
    buffer.setLen(2)
  elif sizeof(value) == 2:
    buffer.setLen(3)
  elif sizeof(value) == 4:
    buffer.setLen(5)
  else:
    buffer.setLen(9)
  let res = LP.putUVarint(buffer, outsize, value)
  if res.isOk():
    buffer.setLen(outsize)
    ok(buffer)
  else:
    err(res.error)