From c893b1f0cb11e49b30fa76b6f81c32b828791004 Mon Sep 17 00:00:00 2001
From: Michael Bradley <michaelsbradleyjr@gmail.com>
Date: Wed, 2 Nov 2022 12:40:28 -0500
Subject: [PATCH] track latest nim-libp2p's master branch (#248)

* [build] track nim-libp2p's unstable branch

* cid formatIt change

* track nim-libp2p-unstable

* don't stringify cid

* Fixed testblockexc.nim

1. Assign price to blocks
2. Delete on node1 cached blocks from node2 before buying them again

* add trace logging

* init pricing info

* remove duplicate price setting

* cid serialization in trace logs

* bumping dht to latest main

* bump

Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
Co-authored-by: Bulat-Ziganshin <bulat.ziganshin@gmail.com>
---
 .gitignore                                    |  1 +
 codex.nim                                     |  2 +-
 codex/blockexchange/engine/engine.nim         |  6 ++++++
 codex/blockexchange/network/networkpeer.nim   |  5 +++++
 codex/discovery.nim                           |  1 +
 codex/node.nim                                |  6 +++---
 codex/rest/api.nim                            |  2 +-
 codex/rng.nim                                 |  9 ++++-----
 codex/streams/asyncstreamwrapper.nim          |  3 ++-
 codex/streams/storestream.nim                 |  2 +-
 .../blockexchange/engine/testblockexc.nim     |  8 ++++++++
 tests/codex/blockexchange/testnetwork.nim     | 11 +++++-----
 tests/codex/helpers.nim                       | 20 ++++++++-----------
 vendor/nim-chronicles                         |  2 +-
 vendor/nim-chronos                            |  2 +-
 vendor/nim-http-utils                         |  2 +-
 vendor/nim-json-serialization                 |  2 +-
 vendor/nim-libp2p                             |  2 +-
 vendor/nim-libp2p-dht                         |  2 +-
 vendor/nim-presto                             |  2 +-
 vendor/nim-serialization                      |  2 +-
 vendor/nim-stew                               |  2 +-
 vendor/nim-websock                            |  2 +-
 23 files changed, 57 insertions(+), 39 deletions(-)

diff --git a/.gitignore b/.gitignore
index 178de456..4ecde076 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,4 @@ nimble.paths
 
 .update.timestamp
 codex.nims
+nimbus-build-system.paths
diff --git a/codex.nim b/codex.nim
index c3ba4087..af39ba03 100644
--- a/codex.nim
+++ b/codex.nim
@@ -90,7 +90,7 @@ when isMainModule:
         notice "Shutting down after having received SIGTERM"
         waitFor server.stop()
 
-      c_signal(SIGTERM, SIGTERMHandler)
+      c_signal(ansi_c.SIGTERM, SIGTERMHandler)
 
     waitFor server.start()
   of StartUpCommand.initNode:
diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim
index fca42f64..3c9c2bdd 100644
--- a/codex/blockexchange/engine/engine.nim
+++ b/codex/blockexchange/engine/engine.nim
@@ -181,6 +181,12 @@ proc blockPresenceHandler*(
 
   for blk in blocks:
     if presence =? Presence.init(blk):
+      logScope:
+        cid   = presence.cid
+        have  = presence.have
+        price = presence.price
+
+      trace "Updating precense"
       peerCtx.updatePresence(presence)
 
   var
diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim
index aee935f9..33d233fd 100644
--- a/codex/blockexchange/network/networkpeer.nim
+++ b/codex/blockexchange/network/networkpeer.nim
@@ -7,6 +7,9 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
+import pkg/upraises
+push: {.upraises: [].}
+
 import pkg/chronos
 import pkg/chronicles
 import pkg/libp2p
@@ -21,6 +24,8 @@ const
   MaxMessageSize = 100 * 1 shl 20 # manifest files can be big
 
 type
+  ConnProvider* = proc(): Future[Connection] {.gcsafe, closure.}
+
   RPCHandler* = proc(peer: NetworkPeer, msg: Message): Future[void] {.gcsafe.}
 
   NetworkPeer* = ref object of RootObj
diff --git a/codex/discovery.nim b/codex/discovery.nim
index 94ae12d5..37b92bd2 100644
--- a/codex/discovery.nim
+++ b/codex/discovery.nim
@@ -22,6 +22,7 @@ import pkg/libp2pdht/discv5/protocol as discv5
 
 import ./rng
 import ./errors
+import ./formats
 
 export discv5
 
diff --git a/codex/node.nim b/codex/node.nim
index f53de6f5..837bdaf7 100644
--- a/codex/node.nim
+++ b/codex/node.nim
@@ -210,14 +210,14 @@ proc store*(
     return failure("Unable to init block from manifest data!")
 
   if isErr (await node.blockStore.putBlock(manifest)):
-    trace "Unable to store manifest", cid = manifest.cid
+    trace "Unable to store manifest", cid = $manifest.cid
     return failure("Unable to store manifest " & $manifest.cid)
 
   without cid =? blockManifest.cid, error:
     trace "Unable to generate manifest Cid!", exc = error.msg
     return failure(error.msg)
 
-  trace "Stored data", manifestCid = manifest.cid,
+  trace "Stored data", manifestCid = $manifest.cid,
                        contentCid = cid,
                        blocks = blockManifest.len
 
@@ -263,7 +263,7 @@ proc requestStorage*(self: CodexNodeRef,
     return failure(error)
 
   if isErr (await self.blockStore.putBlock(encodedBlk)):
-    trace "Unable to store encoded manifest block", cid = encodedBlk.cid
+    trace "Unable to store encoded manifest block", cid = $encodedBlk.cid
     return failure("Unable to store encoded manifest block")
 
   let request = StorageRequest(
diff --git a/codex/rest/api.nim b/codex/rest/api.nim
index f5fef243..ee3765af 100644
--- a/codex/rest/api.nim
+++ b/codex/rest/api.nim
@@ -186,7 +186,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter =
         trace "Excepting streaming blocks", exc = exc.msg
         return RestApiResponse.error(Http500)
       finally:
-        trace "Sent bytes", cid = id.get(), bytes
+        trace "Sent bytes", cid = $id.get(), bytes
         if not stream.isNil:
           await stream.close()
 
diff --git a/codex/rng.nim b/codex/rng.nim
index b1f6266d..19452cd4 100644
--- a/codex/rng.nim
+++ b/codex/rng.nim
@@ -12,11 +12,11 @@ import pkg/upraises
 push: {.upraises: [].}
 
 import pkg/libp2p/crypto/crypto
-import pkg/bearssl
+import pkg/bearssl/rand
 
 type
   RngSampleError = object of CatchableError
-  Rng* = ref BrHmacDrbgContext
+  Rng* = ref HmacDrbgContext
 
 var rng {.threadvar.}: Rng
 
@@ -25,16 +25,15 @@ proc instance*(t: type Rng): Rng =
     rng = newRng()
   rng
 
-# Random helpers: similar as in stdlib, but with BrHmacDrbgContext rng
+# Random helpers: similar as in stdlib, but with HmacDrbgContext rng
 # TODO: Move these somewhere else?
 const randMax = 18_446_744_073_709_551_615'u64
 
 proc rand*(rng: Rng, max: Natural): int =
   if max == 0: return 0
 
-  var x: uint64
   while true:
-    brHmacDrbgGenerate(addr rng[], addr x, csize_t(sizeof(x)))
+    let x = rng[].generate(uint64)
     if x < randMax - (randMax mod (uint64(max) + 1'u64)): # against modulo bias
       return int(x mod (uint64(max) + 1'u64))
 
diff --git a/codex/streams/asyncstreamwrapper.nim b/codex/streams/asyncstreamwrapper.nim
index 81406bc3..a0829b7b 100644
--- a/codex/streams/asyncstreamwrapper.nim
+++ b/codex/streams/asyncstreamwrapper.nim
@@ -7,7 +7,8 @@
 ## This file may not be copied, modified, or distributed except according to
 ## those terms.
 
-{.push raises: [Defect].}
+import pkg/upraises
+push: {.upraises: [].}
 
 import pkg/chronos
 import pkg/chronicles
diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim
index e90d09f1..dee52f30 100644
--- a/codex/streams/storestream.nim
+++ b/codex/streams/storestream.nim
@@ -69,7 +69,7 @@ method readOnce*(
   ## Return how many bytes were actually read before EOF was encountered.
   ## Raise exception if we are already at EOF.
 
-  trace "Reading from manifest", cid = self.manifest.cid.get(), blocks = self.manifest.len
+  trace "Reading from manifest", cid = $self.manifest.cid.get(), blocks = self.manifest.len
   if self.atEof:
     raise newLPStreamEOFError()
 
diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim
index 0f91cf63..f4f3f408 100644
--- a/tests/codex/blockexchange/engine/testblockexc.nim
+++ b/tests/codex/blockexchange/engine/testblockexc.nim
@@ -15,6 +15,7 @@ import pkg/codex/chunker
 import pkg/codex/discovery
 import pkg/codex/blocktype as bt
 
+import ../../examples
 import ../../helpers
 
 suite "NetworkStore engine - 2 nodes":
@@ -59,6 +60,9 @@ suite "NetworkStore engine - 2 nodes":
     pendingBlocks1 = blocks2.mapIt( nodeCmps1.pendingBlocks.getWantHandle( it.cid ) )
     pendingBlocks2 = blocks1.mapIt( nodeCmps2.pendingBlocks.getWantHandle( it.cid ) )
 
+    pricing1 = Pricing.example()
+    pricing2 = Pricing.example()
+
     pricing1.address = nodeCmps1.wallet.address
     pricing2.address = nodeCmps2.wallet.address
     nodeCmps1.engine.pricing = pricing1.some
@@ -144,6 +148,10 @@ suite "NetworkStore engine - 2 nodes":
       .withTimeout(100.millis) # should succeed
 
   test "Should receive payments for blocks that were sent":
+    # delete on node1 cached blocks from node2
+    discard await allFinished(
+      blocks2.mapIt( nodeCmps1.networkStore.delBlock(it.cid) ))
+
     let blocks = await allFinished(
       blocks2.mapIt( nodeCmps1.networkStore.getBlock(it.cid) ))
 
diff --git a/tests/codex/blockexchange/testnetwork.nim b/tests/codex/blockexchange/testnetwork.nim
index 1df53afb..f26f5ae6 100644
--- a/tests/codex/blockexchange/testnetwork.nim
+++ b/tests/codex/blockexchange/testnetwork.nim
@@ -158,9 +158,6 @@ suite "Network - Senders":
     done = newFuture[void]()
     switch1 = newStandardSwitch()
     switch2 = newStandardSwitch()
-    await switch1.start()
-    await switch2.start()
-
     network1 = BlockExcNetwork.new(
       switch = switch1)
     switch1.mount(network1)
@@ -169,6 +166,9 @@ suite "Network - Senders":
       switch = switch2)
     switch2.mount(network2)
 
+    await switch1.start()
+    await switch2.start()
+
     await switch1.connect(
       switch2.peerInfo.peerId,
       switch2.peerInfo.addrs)
@@ -271,8 +271,6 @@ suite "Network - Test Limits":
     done = newFuture[void]()
     switch1 = newStandardSwitch()
     switch2 = newStandardSwitch()
-    await switch1.start()
-    await switch2.start()
 
     network1 = BlockExcNetwork.new(
       switch = switch1,
@@ -283,6 +281,9 @@ suite "Network - Test Limits":
       switch = switch2)
     switch2.mount(network2)
 
+    await switch1.start()
+    await switch2.start()
+
     await switch1.connect(
       switch2.peerInfo.peerId,
       switch2.peerInfo.addrs)
diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim
index 475bf7a1..6d431870 100644
--- a/tests/codex/helpers.nim
+++ b/tests/codex/helpers.nim
@@ -37,12 +37,11 @@ proc corruptBlocks*(
   manifest: Manifest,
   blks, bytes: int): Future[seq[int]] {.async.} =
   var pos: seq[int]
-  while true:
-    if pos.len >= blks:
-      break
 
-    var i = -1
-    if (i = Rng.instance.rand(manifest.len - 1); pos.find(i) >= 0):
+  doAssert blks < manifest.len
+  while pos.len < blks:
+    let i = Rng.instance.rand(manifest.len - 1)
+    if pos.find(i) >= 0:
       continue
 
     pos.add(i)
@@ -50,15 +49,12 @@ proc corruptBlocks*(
       blk = (await store.getBlock(manifest[i])).tryGet()
       bytePos: seq[int]
 
-    while true:
-      if bytePos.len > bytes:
-        break
-
-      var ii = -1
-      if (ii = Rng.instance.rand(blk.data.len - 1); bytePos.find(ii) >= 0):
+    doAssert bytes < blk.data.len
+    while bytePos.len <= bytes:
+      let ii = Rng.instance.rand(blk.data.len - 1)
+      if bytePos.find(ii) >= 0:
         continue
 
       bytePos.add(ii)
       blk.data[ii] = byte 0
-
   return pos
diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles
index 16820963..7631f7b2 160000
--- a/vendor/nim-chronicles
+++ b/vendor/nim-chronicles
@@ -1 +1 @@
-Subproject commit 1682096306ddba8185dcfac360a8c3f952d721e4
+Subproject commit 7631f7b2ee03398cb1512a79923264e8f9410af6
diff --git a/vendor/nim-chronos b/vendor/nim-chronos
index 17fed89c..8e826337 160000
--- a/vendor/nim-chronos
+++ b/vendor/nim-chronos
@@ -1 +1 @@
-Subproject commit 17fed89c99beac5a92d3668d0d3e9b0e4ac13936
+Subproject commit 8e8263370b51bc9b71916273e3eb956053532c4f
diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils
index 689da19e..e88e231d 160000
--- a/vendor/nim-http-utils
+++ b/vendor/nim-http-utils
@@ -1 +1 @@
-Subproject commit 689da19e9e9cfff4ced85e2b25c6b2b5598ed079
+Subproject commit e88e231dfcef4585fe3b2fbd9b664dbd28a88040
diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization
index c5f0e246..e5b18fb7 160000
--- a/vendor/nim-json-serialization
+++ b/vendor/nim-json-serialization
@@ -1 +1 @@
-Subproject commit c5f0e2465e8375dfc7aa0f56ccef67cb680bc6b0
+Subproject commit e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4
diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p
index eeb3c210..a3e9d1ed 160000
--- a/vendor/nim-libp2p
+++ b/vendor/nim-libp2p
@@ -1 +1 @@
-Subproject commit eeb3c210a37408716b6a8b45f578adf87610cef2
+Subproject commit a3e9d1ed80c048cd5abc839cbe0863cefcedc702
diff --git a/vendor/nim-libp2p-dht b/vendor/nim-libp2p-dht
index d6d255b4..e4e7a3e1 160000
--- a/vendor/nim-libp2p-dht
+++ b/vendor/nim-libp2p-dht
@@ -1 +1 @@
-Subproject commit d6d255b4b5d6a4fa56db0eb6677ed7391cbb4897
+Subproject commit e4e7a3e11fe635de3f15e37164b3ace96f588993
diff --git a/vendor/nim-presto b/vendor/nim-presto
index 962bb588..3984431d 160000
--- a/vendor/nim-presto
+++ b/vendor/nim-presto
@@ -1 +1 @@
-Subproject commit 962bb588d19c7180e39f0d9f18131e75861bab20
+Subproject commit 3984431dc0fc829eb668e12e57e90542b041d298
diff --git a/vendor/nim-serialization b/vendor/nim-serialization
index fcd0eada..493d18b8 160000
--- a/vendor/nim-serialization
+++ b/vendor/nim-serialization
@@ -1 +1 @@
-Subproject commit fcd0eadadde0ee000a63df8ab21dc4e9f015a790
+Subproject commit 493d18b8292fc03aa4f835fd825dea1183f97466
diff --git a/vendor/nim-stew b/vendor/nim-stew
index 6ad35b87..0c379cf1 160000
--- a/vendor/nim-stew
+++ b/vendor/nim-stew
@@ -1 +1 @@
-Subproject commit 6ad35b876fb6ebe0dfee0f697af173acc47906ee
+Subproject commit 0c379cf1d8d3d9db07af108cc78ff542b2105914
diff --git a/vendor/nim-websock b/vendor/nim-websock
index a697e358..7b2ed397 160000
--- a/vendor/nim-websock
+++ b/vendor/nim-websock
@@ -1 +1 @@
-Subproject commit a697e3585d583ab6b91a159ea7d023461002c927
+Subproject commit 7b2ed397d6e4c37ea4df08ae82aeac7ff04cd180