track latest nim-libp2p's master branch (#248)

* [build] track nim-libp2p's unstable branch

* cid formatIt change

* track nim-libp2p-unstable

* don't stringify cid

* Fixed testblockexc.nim

1. Assign price to blocks
2. Delete on node1 cached blocks from node2 before buying them again

* add trace logging

* init pricing info

* remove duplicate price setting

* cid serialization in trace logs

* bumping dht to latest main

* bump

Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
Co-authored-by: Bulat-Ziganshin <bulat.ziganshin@gmail.com>

.gitignore

@@ -29,3 +29,4 @@ nimble.paths
 
 .update.timestamp
 codex.nims
+nimbus-build-system.paths

codex.nim

@@ -90,7 +90,7 @@ when isMainModule:
         notice "Shutting down after having received SIGTERM"
         waitFor server.stop()
 
-      c_signal(SIGTERM, SIGTERMHandler)
+      c_signal(ansi_c.SIGTERM, SIGTERMHandler)
 
     waitFor server.start()
   of StartUpCommand.initNode:

codex/blockexchange/engine/engine.nim

@@ -181,6 +181,12 @@ proc blockPresenceHandler*(
 
   for blk in blocks:
     if presence =? Presence.init(blk):
+      logScope:
+        cid   = presence.cid
+        have  = presence.have
+        price = presence.price
+
+      trace "Updating precense"
       peerCtx.updatePresence(presence)
 
   var

codex/blockexchange/network/networkpeer.nim

@@ -7,6 +7,9 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
+import pkg/upraises
+push: {.upraises: [].}
+
 import pkg/chronos
 import pkg/chronicles
 import pkg/libp2p
@@ -21,6 +24,8 @@ const
   MaxMessageSize = 100 * 1 shl 20 # manifest files can be big
 
 type
+  ConnProvider* = proc(): Future[Connection] {.gcsafe, closure.}
+
   RPCHandler* = proc(peer: NetworkPeer, msg: Message): Future[void] {.gcsafe.}
 
   NetworkPeer* = ref object of RootObj

codex/discovery.nim

@@ -22,6 +22,7 @@ import pkg/libp2pdht/discv5/protocol as discv5
 
 import ./rng
 import ./errors
+import ./formats
 
 export discv5
 

codex/node.nim

@@ -210,14 +210,14 @@ proc store*(
     return failure("Unable to init block from manifest data!")
 
   if isErr (await node.blockStore.putBlock(manifest)):
-    trace "Unable to store manifest", cid = manifest.cid
+    trace "Unable to store manifest", cid = $manifest.cid
     return failure("Unable to store manifest " & $manifest.cid)
 
   without cid =? blockManifest.cid, error:
     trace "Unable to generate manifest Cid!", exc = error.msg
     return failure(error.msg)
 
-  trace "Stored data", manifestCid = manifest.cid,
+  trace "Stored data", manifestCid = $manifest.cid,
                        contentCid = cid,
                        blocks = blockManifest.len
 
@@ -263,7 +263,7 @@ proc requestStorage*(self: CodexNodeRef,
     return failure(error)
 
   if isErr (await self.blockStore.putBlock(encodedBlk)):
-    trace "Unable to store encoded manifest block", cid = encodedBlk.cid
+    trace "Unable to store encoded manifest block", cid = $encodedBlk.cid
     return failure("Unable to store encoded manifest block")
 
   let request = StorageRequest(

codex/rest/api.nim

@@ -186,7 +186,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
         trace "Excepting streaming blocks", exc = exc.msg
         return RestApiResponse.error(Http500)
       finally:
-        trace "Sent bytes", cid = id.get(), bytes
+        trace "Sent bytes", cid = $id.get(), bytes
         if not stream.isNil:
           await stream.close()
 

codex/rng.nim

@@ -12,11 +12,11 @@ import pkg/upraises
 push: {.upraises: [].}
 
 import pkg/libp2p/crypto/crypto
-import pkg/bearssl
+import pkg/bearssl/rand
 
 type
   RngSampleError = object of CatchableError
-  Rng* = ref BrHmacDrbgContext
+  Rng* = ref HmacDrbgContext
 
 var rng {.threadvar.}: Rng
 
@@ -25,16 +25,15 @@ proc instance*(t: type Rng): Rng =
     rng = newRng()
   rng
 
-# Random helpers: similar as in stdlib, but with BrHmacDrbgContext rng
+# Random helpers: similar as in stdlib, but with HmacDrbgContext rng
 # TODO: Move these somewhere else?
 const randMax = 18_446_744_073_709_551_615'u64
 
 proc rand*(rng: Rng, max: Natural): int =
   if max == 0: return 0
 
-  var x: uint64
   while true:
-    brHmacDrbgGenerate(addr rng[], addr x, csize_t(sizeof(x)))
+    let x = rng[].generate(uint64)
     if x < randMax - (randMax mod (uint64(max) + 1'u64)): # against modulo bias
       return int(x mod (uint64(max) + 1'u64))
 

codex/streams/asyncstreamwrapper.nim

@@ -7,7 +7,8 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
-{.push raises: [Defect].}
+import pkg/upraises
+push: {.upraises: [].}
 
 import pkg/chronos
 import pkg/chronicles

codex/streams/storestream.nim

@@ -69,7 +69,7 @@ method readOnce*(
   ## Return how many bytes were actually read before EOF was encountered.
   ## Raise exception if we are already at EOF.
 
-  trace "Reading from manifest", cid = self.manifest.cid.get(), blocks = self.manifest.len
+  trace "Reading from manifest", cid = $self.manifest.cid.get(), blocks = self.manifest.len
   if self.atEof:
     raise newLPStreamEOFError()
 

tests/codex/blockexchange/engine/testblockexc.nim

@@ -15,6 +15,7 @@ import pkg/codex/chunker
 import pkg/codex/discovery
 import pkg/codex/blocktype as bt
 
+import ../../examples
 import ../../helpers
 
 suite "NetworkStore engine - 2 nodes":
@@ -59,6 +60,9 @@ suite "NetworkStore engine - 2 nodes":
     pendingBlocks1 = blocks2.mapIt( nodeCmps1.pendingBlocks.getWantHandle( it.cid ) )
     pendingBlocks2 = blocks1.mapIt( nodeCmps2.pendingBlocks.getWantHandle( it.cid ) )
 
+    pricing1 = Pricing.example()
+    pricing2 = Pricing.example()
+
     pricing1.address = nodeCmps1.wallet.address
     pricing2.address = nodeCmps2.wallet.address
     nodeCmps1.engine.pricing = pricing1.some
@@ -144,6 +148,10 @@ suite "NetworkStore engine - 2 nodes":
       .withTimeout(100.millis) # should succeed
 
   test "Should receive payments for blocks that were sent":
+    # delete on node1 cached blocks from node2
+    discard await allFinished(
+      blocks2.mapIt( nodeCmps1.networkStore.delBlock(it.cid) ))
+
     let blocks = await allFinished(
       blocks2.mapIt( nodeCmps1.networkStore.getBlock(it.cid) ))
 

tests/codex/blockexchange/testnetwork.nim

@@ -158,9 +158,6 @@ suite "Network - Senders":
     done = newFuture[void]()
     switch1 = newStandardSwitch()
     switch2 = newStandardSwitch()
-    await switch1.start()
-    await switch2.start()
-
     network1 = BlockExcNetwork.new(
       switch = switch1)
     switch1.mount(network1)
@@ -169,6 +166,9 @@ suite "Network - Senders":
       switch = switch2)
     switch2.mount(network2)
 
+    await switch1.start()
+    await switch2.start()
+
     await switch1.connect(
       switch2.peerInfo.peerId,
       switch2.peerInfo.addrs)
@@ -271,8 +271,6 @@ suite "Network - Test Limits":
     done = newFuture[void]()
     switch1 = newStandardSwitch()
     switch2 = newStandardSwitch()
-    await switch1.start()
-    await switch2.start()
 
     network1 = BlockExcNetwork.new(
       switch = switch1,
@@ -283,6 +281,9 @@ suite "Network - Test Limits":
       switch = switch2)
     switch2.mount(network2)
 
+    await switch1.start()
+    await switch2.start()
+
     await switch1.connect(
       switch2.peerInfo.peerId,
       switch2.peerInfo.addrs)

tests/codex/helpers.nim

@@ -37,12 +37,11 @@ proc corruptBlocks*(
   manifest: Manifest,
   blks, bytes: int): Future[seq[int]] {.async.} =
   var pos: seq[int]
-  while true:
-    if pos.len >= blks:
-      break
 
-    var i = -1
+  doAssert blks < manifest.len
-    if (i = Rng.instance.rand(manifest.len - 1); pos.find(i) >= 0):
+  while pos.len < blks:
+    let i = Rng.instance.rand(manifest.len - 1)
+    if pos.find(i) >= 0:
       continue
 
     pos.add(i)
@@ -50,15 +49,12 @@ proc corruptBlocks*(
       blk = (await store.getBlock(manifest[i])).tryGet()
       bytePos: seq[int]
 
-    while true:
+    doAssert bytes < blk.data.len
-      if bytePos.len > bytes:
+    while bytePos.len <= bytes:
-        break
+      let ii = Rng.instance.rand(blk.data.len - 1)
-
+      if bytePos.find(ii) >= 0:
-      var ii = -1
-      if (ii = Rng.instance.rand(blk.data.len - 1); bytePos.find(ii) >= 0):
         continue
 
       bytePos.add(ii)
       blk.data[ii] = byte 0
-
   return pos

vendor/nim-chronicles

@@ -1 +1 @@
-Subproject commit 1682096306ddba8185dcfac360a8c3f952d721e4
+Subproject commit 7631f7b2ee03398cb1512a79923264e8f9410af6

vendor/nim-chronos

@@ -1 +1 @@
-Subproject commit 17fed89c99beac5a92d3668d0d3e9b0e4ac13936
+Subproject commit 8e8263370b51bc9b71916273e3eb956053532c4f

vendor/nim-http-utils

@@ -1 +1 @@
-Subproject commit 689da19e9e9cfff4ced85e2b25c6b2b5598ed079
+Subproject commit e88e231dfcef4585fe3b2fbd9b664dbd28a88040

vendor/nim-json-serialization

@@ -1 +1 @@
-Subproject commit c5f0e2465e8375dfc7aa0f56ccef67cb680bc6b0
+Subproject commit e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4

vendor/nim-libp2p

@@ -1 +1 @@
-Subproject commit eeb3c210a37408716b6a8b45f578adf87610cef2
+Subproject commit a3e9d1ed80c048cd5abc839cbe0863cefcedc702

vendor/nim-libp2p-dht

@@ -1 +1 @@
-Subproject commit d6d255b4b5d6a4fa56db0eb6677ed7391cbb4897
+Subproject commit e4e7a3e11fe635de3f15e37164b3ace96f588993

vendor/nim-presto

@@ -1 +1 @@
-Subproject commit 962bb588d19c7180e39f0d9f18131e75861bab20
+Subproject commit 3984431dc0fc829eb668e12e57e90542b041d298

vendor/nim-serialization

@@ -1 +1 @@
-Subproject commit fcd0eadadde0ee000a63df8ab21dc4e9f015a790
+Subproject commit 493d18b8292fc03aa4f835fd825dea1183f97466

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 6ad35b876fb6ebe0dfee0f697af173acc47906ee
+Subproject commit 0c379cf1d8d3d9db07af108cc78ff542b2105914

vendor/nim-websock

@@ -1 +1 @@
-Subproject commit a697e3585d583ab6b91a159ea7d023461002c927
+Subproject commit 7b2ed397d6e4c37ea4df08ae82aeac7ff04cd180