Fix for decoding large manifests (#479)

.gitignore

@@ -35,3 +35,4 @@ codex.nims
 nimbus-build-system.paths
 docker/hostdatadir
 docker/prometheus-data
+.DS_Store

codex/blockexchange/network/network.nim

@@ -84,6 +84,8 @@ proc send*(b: BlockExcNetwork, id: PeerId, msg: pb.Message) {.async.} =
       await b.inflightSema.acquire()
       trace "Sending message to peer", peer = id
       await peer[].send(msg)
+    except CatchableError as err:
+      error "Error sending message", peer = id, msg = err.msg
     finally:
       b.inflightSema.release()
   do:

codex/blockexchange/network/networkpeer.nim

@@ -15,14 +15,12 @@ import pkg/chronicles
 import pkg/libp2p
 
 import ../protobuf/blockexc
+import ../protobuf/message
 import ../../errors
 
 logScope:
   topics = "codex blockexcnetworkpeer"
 
-const
-  MaxMessageSize = 100 * 1 shl 20 # manifest files can be big
-
 type
   ConnProvider* = proc(): Future[Connection] {.gcsafe, closure.}
 
@@ -45,12 +43,12 @@ proc readLoop*(b: NetworkPeer, conn: Connection) {.async.} =
   try:
     while not conn.atEof or not conn.closed:
       let
-        data = await conn.readLp(MaxMessageSize)
+        data = await conn.readLp(MaxMessageSize.int)
         msg = Message.protobufDecode(data).mapFailure().tryGet()
       trace "Got message for peer", peer = b.id
       await b.handler(b, msg)
-  except CatchableError as exc:
+  except CatchableError as err:
-    trace "Exception in blockexc read loop", exc = exc.msg
+    warn "Exception in blockexc read loop", msg = err.msg
   finally:
     await conn.close()
 

codex/blockexchange/protobuf/message.nim

@@ -5,6 +5,11 @@
 
 import pkg/libp2p/protobuf/minprotobuf
 
+import ../../units
+
+const
+  MaxBlockSize* = 100.MiBs.uint
+  MaxMessageSize* = 100.MiBs.uint
 
 type
   WantType* = enum
@@ -72,7 +77,7 @@ proc write*(pb: var ProtoBuffer, field: int, value: Wantlist) =
   pb.write(field, ipb)
 
 proc write*(pb: var ProtoBuffer, field: int, value: Block) =
-  var ipb = initProtoBuffer()
+  var ipb = initProtoBuffer(maxSize = MaxBlockSize)
   ipb.write(1, value.prefix)
   ipb.write(2, value.data)
   ipb.finish()
@@ -99,7 +104,7 @@ proc write*(pb: var ProtoBuffer, field: int, value: StateChannelUpdate) =
   pb.write(field, ipb)
 
 proc protobufEncode*(value: Message): seq[byte] =
-  var ipb = initProtoBuffer()
+  var ipb = initProtoBuffer(maxSize = MaxMessageSize)
   ipb.write(1, value.wantlist)
   for v in value.payload:
     ipb.write(3, v)
@@ -175,14 +180,14 @@ proc decode*(_: type StateChannelUpdate, pb: ProtoBuffer): ProtoResult[StateChan
 proc protobufDecode*(_: type Message, msg: seq[byte]): ProtoResult[Message] =
   var
     value = Message()
-    pb = initProtoBuffer(msg)
+    pb = initProtoBuffer(msg, maxSize = MaxMessageSize)
     ipb: ProtoBuffer
     sublist: seq[seq[byte]]
   if ? pb.getField(1, ipb):
     value.wantlist = ? Wantlist.decode(ipb)
   if ? pb.getRepeatedField(3, sublist):
     for item in sublist:
-      value.payload.add(? Block.decode(initProtoBuffer(item)))
+      value.payload.add(? Block.decode(initProtoBuffer(item, maxSize = MaxBlockSize)))
   if ? pb.getRepeatedField(4, sublist):
     for item in sublist:
       value.blockPresences.add(? BlockPresence.decode(initProtoBuffer(item)))

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit a3e9d1ed80c048cd5abc839cbe0863cefcedc702
+Subproject commit 8c2eca18dcc538c57a8fbc0c53fd0b9d24d56cff