From 4e191a06ac0703e65e3789f3deda0ff829f4afad Mon Sep 17 00:00:00 2001
From: Jacek Sieka <jacek@status.im>
Date: Mon, 14 Dec 2020 15:09:26 +0100
Subject: [PATCH] use stew/leb128

* 3 varint parsers -> 1
* 1/3 memory usage / allocations when writing stuff to libp2p
---
 beacon_chain/eth2_network.nim          | 21 ++++++++++-----------
 beacon_chain/ssz/ssz_serialization.nim | 15 ++++-----------
 vendor/nim-libp2p                      |  2 +-
 vendor/nim-snappy                      |  2 +-
 vendor/nim-stew                        |  2 +-
 5 files changed, 17 insertions(+), 25 deletions(-)

diff --git a/beacon_chain/eth2_network.nim b/beacon_chain/eth2_network.nim
index 86f3248ee..a332461c1 100644
--- a/beacon_chain/eth2_network.nim
+++ b/beacon_chain/eth2_network.nim
@@ -4,15 +4,14 @@ import
   std/options as stdOptions,
 
   # Status libs
-  stew/[varints, base58, endians2, results, byteutils, io2], bearssl,
+  stew/[leb128, base58, endians2, results, byteutils, io2], bearssl,
   stew/shims/net as stewNet,
   stew/shims/[macros, tables],
   faststreams/[inputs, outputs, buffers], snappy, snappy/framing,
   json_serialization, json_serialization/std/[net, options],
   chronos, chronicles, metrics,
-  # TODO: create simpler to use libp2p modules that use re-exports
   libp2p/[switch, peerinfo,
-          multiaddress, multicodec, crypto/crypto, crypto/secp,
+          multiaddress, crypto/crypto, crypto/secp,
           protocols/identify, protocols/protocol],
   libp2p/muxers/muxer, libp2p/muxers/mplex/mplex,
   libp2p/transports/[transport, tcptransport],
@@ -496,8 +495,8 @@ proc writeChunk*(conn: Connection,
   if responseCode.isSome:
     output.write byte(responseCode.get)
 
-  output.write varintBytes(payload.lenu64)
-  output.write(framingFormatCompress payload)
+  output.write toBytes(payload.lenu64, Leb128).toOpenArray()
+  framingFormatCompress(output, payload)
 
   conn.write(output.getOutput)
 
@@ -505,14 +504,14 @@ template errorMsgLit(x: static string): ErrorMsg =
   const val = ErrorMsg toBytes(x)
   val
 
-func formatErrorMsg(msg: ErrorMSg): string =
-  let candidate = string.fromBytes(asSeq(msg))
-  for c in candidate:
-    # TODO UTF-8 - but let's start with ASCII
-    if ord(c) < 32 or ord(c) > 127:
+func formatErrorMsg(msg: ErrorMsg): string =
+  # ErrorMsg "usually" contains a human-readable string - we'll try to parse it
+  # as ASCII and return hex if that fails
+  for c in msg:
+    if c < 32 or c > 127:
       return byteutils.toHex(asSeq(msg))
 
-  return candidate
+  string.fromBytes(asSeq(msg))
 
 proc sendErrorResponse(peer: Peer,
                        conn: Connection,
diff --git a/beacon_chain/ssz/ssz_serialization.nim b/beacon_chain/ssz/ssz_serialization.nim
index 831167a79..955cb2ea1 100644
--- a/beacon_chain/ssz/ssz_serialization.nim
+++ b/beacon_chain/ssz/ssz_serialization.nim
@@ -15,7 +15,7 @@
 
 import
   std/[typetraits, options],
-  stew/[bitops2, endians2, objects],
+  stew/[bitops2, endians2, leb128, objects],
   serialization, serialization/testing/tracing,
   ../spec/[digest, datatypes],
   ./bytes_reader, ./bitseqs, ./types, ./spec_types
@@ -219,18 +219,11 @@ func sszSize*(value: auto): int {.gcsafe, raises: [Defect].} =
     unsupported T
 
 proc writeValue*[T](w: var SszWriter, x: SizePrefixed[T]) {.raises: [Defect, IOError].} =
-  var cursor = w.stream.delayVarSizeWrite(10)
+  var cursor = w.stream.delayVarSizeWrite(Leb128.maxLen(uint64))
   let initPos = w.stream.pos
   w.writeValue T(x)
-  let length = uint64(w.stream.pos - initPos)
-  when false:
-    discard
-    # TODO varintBytes is sub-optimal at the moment
-    # cursor.writeAndFinalize length.varintBytes
-  else:
-    var buf: VarintBuffer
-    buf.writeVarint length
-    cursor.finalWrite buf.writtenBytes
+  let length = toBytes(uint64(w.stream.pos - initPos), Leb128)
+  cursor.finalWrite length.toOpenArray()
 
 proc readValue*[T](r: var SszReader, val: var T) {.raises: [Defect, MalformedSszError, SszSizeMismatchError, IOError].} =
   when isFixedSize(T):
diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p
index 1befeb8c2..2f9064566 160000
--- a/vendor/nim-libp2p
+++ b/vendor/nim-libp2p
@@ -1 +1 @@
-Subproject commit 1befeb8c2e74425787d7bfd7b53d6c60302161c7
+Subproject commit 2f90645669d18626f928300b918dc14664e00051
diff --git a/vendor/nim-snappy b/vendor/nim-snappy
index 8455b825e..b2f48c0fd 160000
--- a/vendor/nim-snappy
+++ b/vendor/nim-snappy
@@ -1 +1 @@
-Subproject commit 8455b825e50be706eb689f7629dac7098d2d5da5
+Subproject commit b2f48c0fd331a72fc917cd9cd7fadea683ca9de2
diff --git a/vendor/nim-stew b/vendor/nim-stew
index e15c1ae01..53979b7c5 160000
--- a/vendor/nim-stew
+++ b/vendor/nim-stew
@@ -1 +1 @@
-Subproject commit e15c1ae01269b1c53e1c9d80741b6d8929fc4b75
+Subproject commit 53979b7c5a5d18b015b0e5296d7d02c72e92eb5a