From c893b1f0cb11e49b30fa76b6f81c32b828791004 Mon Sep 17 00:00:00 2001 From: Michael Bradley Date: Wed, 2 Nov 2022 12:40:28 -0500 Subject: [PATCH] track latest nim-libp2p's master branch (#248) * [build] track nim-libp2p's unstable branch * cid formatIt change * track nim-libp2p-unstable * don't stringify cid * Fixed testblockexc.nim 1. Assign price to blocks 2. Delete on node1 cached blocks from node2 before buying them again * add trace logging * init pricing info * remove duplicate price setting * cid serialization in trace logs * bumping dht to latest main * bump Co-authored-by: Dmitriy Ryajov Co-authored-by: Bulat-Ziganshin --- .gitignore | 1 + codex.nim | 2 +- codex/blockexchange/engine/engine.nim | 6 ++++++ codex/blockexchange/network/networkpeer.nim | 5 +++++ codex/discovery.nim | 1 + codex/node.nim | 6 +++--- codex/rest/api.nim | 2 +- codex/rng.nim | 9 ++++----- codex/streams/asyncstreamwrapper.nim | 3 ++- codex/streams/storestream.nim | 2 +- .../blockexchange/engine/testblockexc.nim | 8 ++++++++ tests/codex/blockexchange/testnetwork.nim | 11 +++++----- tests/codex/helpers.nim | 20 ++++++++----------- vendor/nim-chronicles | 2 +- vendor/nim-chronos | 2 +- vendor/nim-http-utils | 2 +- vendor/nim-json-serialization | 2 +- vendor/nim-libp2p | 2 +- vendor/nim-libp2p-dht | 2 +- vendor/nim-presto | 2 +- vendor/nim-serialization | 2 +- vendor/nim-stew | 2 +- vendor/nim-websock | 2 +- 23 files changed, 57 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 178de456..4ecde076 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ nimble.paths .update.timestamp codex.nims +nimbus-build-system.paths diff --git a/codex.nim b/codex.nim index c3ba4087..af39ba03 100644 --- a/codex.nim +++ b/codex.nim @@ -90,7 +90,7 @@ when isMainModule: notice "Shutting down after having received SIGTERM" waitFor server.stop() - c_signal(SIGTERM, SIGTERMHandler) + c_signal(ansi_c.SIGTERM, SIGTERMHandler) waitFor server.start() of StartUpCommand.initNode: diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index fca42f64..3c9c2bdd 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -181,6 +181,12 @@ proc blockPresenceHandler*( for blk in blocks: if presence =? Presence.init(blk): + logScope: + cid = presence.cid + have = presence.have + price = presence.price + + trace "Updating precense" peerCtx.updatePresence(presence) var diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index aee935f9..33d233fd 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -7,6 +7,9 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. +import pkg/upraises +push: {.upraises: [].} + import pkg/chronos import pkg/chronicles import pkg/libp2p @@ -21,6 +24,8 @@ const MaxMessageSize = 100 * 1 shl 20 # manifest files can be big type + ConnProvider* = proc(): Future[Connection] {.gcsafe, closure.} + RPCHandler* = proc(peer: NetworkPeer, msg: Message): Future[void] {.gcsafe.} NetworkPeer* = ref object of RootObj diff --git a/codex/discovery.nim b/codex/discovery.nim index 94ae12d5..37b92bd2 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -22,6 +22,7 @@ import pkg/libp2pdht/discv5/protocol as discv5 import ./rng import ./errors +import ./formats export discv5 diff --git a/codex/node.nim b/codex/node.nim index f53de6f5..837bdaf7 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -210,14 +210,14 @@ proc store*( return failure("Unable to init block from manifest data!") if isErr (await node.blockStore.putBlock(manifest)): - trace "Unable to store manifest", cid = manifest.cid + trace "Unable to store manifest", cid = $manifest.cid return failure("Unable to store manifest " & $manifest.cid) without cid =? blockManifest.cid, error: trace "Unable to generate manifest Cid!", exc = error.msg return failure(error.msg) - trace "Stored data", manifestCid = manifest.cid, + trace "Stored data", manifestCid = $manifest.cid, contentCid = cid, blocks = blockManifest.len @@ -263,7 +263,7 @@ proc requestStorage*(self: CodexNodeRef, return failure(error) if isErr (await self.blockStore.putBlock(encodedBlk)): - trace "Unable to store encoded manifest block", cid = encodedBlk.cid + trace "Unable to store encoded manifest block", cid = $encodedBlk.cid return failure("Unable to store encoded manifest block") let request = StorageRequest( diff --git a/codex/rest/api.nim b/codex/rest/api.nim index f5fef243..ee3765af 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -186,7 +186,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = trace "Excepting streaming blocks", exc = exc.msg return RestApiResponse.error(Http500) finally: - trace "Sent bytes", cid = id.get(), bytes + trace "Sent bytes", cid = $id.get(), bytes if not stream.isNil: await stream.close() diff --git a/codex/rng.nim b/codex/rng.nim index b1f6266d..19452cd4 100644 --- a/codex/rng.nim +++ b/codex/rng.nim @@ -12,11 +12,11 @@ import pkg/upraises push: {.upraises: [].} import pkg/libp2p/crypto/crypto -import pkg/bearssl +import pkg/bearssl/rand type RngSampleError = object of CatchableError - Rng* = ref BrHmacDrbgContext + Rng* = ref HmacDrbgContext var rng {.threadvar.}: Rng @@ -25,16 +25,15 @@ proc instance*(t: type Rng): Rng = rng = newRng() rng -# Random helpers: similar as in stdlib, but with BrHmacDrbgContext rng +# Random helpers: similar as in stdlib, but with HmacDrbgContext rng # TODO: Move these somewhere else? const randMax = 18_446_744_073_709_551_615'u64 proc rand*(rng: Rng, max: Natural): int = if max == 0: return 0 - var x: uint64 while true: - brHmacDrbgGenerate(addr rng[], addr x, csize_t(sizeof(x))) + let x = rng[].generate(uint64) if x < randMax - (randMax mod (uint64(max) + 1'u64)): # against modulo bias return int(x mod (uint64(max) + 1'u64)) diff --git a/codex/streams/asyncstreamwrapper.nim b/codex/streams/asyncstreamwrapper.nim index 81406bc3..a0829b7b 100644 --- a/codex/streams/asyncstreamwrapper.nim +++ b/codex/streams/asyncstreamwrapper.nim @@ -7,7 +7,8 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -{.push raises: [Defect].} +import pkg/upraises +push: {.upraises: [].} import pkg/chronos import pkg/chronicles diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index e90d09f1..dee52f30 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -69,7 +69,7 @@ method readOnce*( ## Return how many bytes were actually read before EOF was encountered. ## Raise exception if we are already at EOF. - trace "Reading from manifest", cid = self.manifest.cid.get(), blocks = self.manifest.len + trace "Reading from manifest", cid = $self.manifest.cid.get(), blocks = self.manifest.len if self.atEof: raise newLPStreamEOFError() diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim index 0f91cf63..f4f3f408 100644 --- a/tests/codex/blockexchange/engine/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -15,6 +15,7 @@ import pkg/codex/chunker import pkg/codex/discovery import pkg/codex/blocktype as bt +import ../../examples import ../../helpers suite "NetworkStore engine - 2 nodes": @@ -59,6 +60,9 @@ suite "NetworkStore engine - 2 nodes": pendingBlocks1 = blocks2.mapIt( nodeCmps1.pendingBlocks.getWantHandle( it.cid ) ) pendingBlocks2 = blocks1.mapIt( nodeCmps2.pendingBlocks.getWantHandle( it.cid ) ) + pricing1 = Pricing.example() + pricing2 = Pricing.example() + pricing1.address = nodeCmps1.wallet.address pricing2.address = nodeCmps2.wallet.address nodeCmps1.engine.pricing = pricing1.some @@ -144,6 +148,10 @@ suite "NetworkStore engine - 2 nodes": .withTimeout(100.millis) # should succeed test "Should receive payments for blocks that were sent": + # delete on node1 cached blocks from node2 + discard await allFinished( + blocks2.mapIt( nodeCmps1.networkStore.delBlock(it.cid) )) + let blocks = await allFinished( blocks2.mapIt( nodeCmps1.networkStore.getBlock(it.cid) )) diff --git a/tests/codex/blockexchange/testnetwork.nim b/tests/codex/blockexchange/testnetwork.nim index 1df53afb..f26f5ae6 100644 --- a/tests/codex/blockexchange/testnetwork.nim +++ b/tests/codex/blockexchange/testnetwork.nim @@ -158,9 +158,6 @@ suite "Network - Senders": done = newFuture[void]() switch1 = newStandardSwitch() switch2 = newStandardSwitch() - await switch1.start() - await switch2.start() - network1 = BlockExcNetwork.new( switch = switch1) switch1.mount(network1) @@ -169,6 +166,9 @@ suite "Network - Senders": switch = switch2) switch2.mount(network2) + await switch1.start() + await switch2.start() + await switch1.connect( switch2.peerInfo.peerId, switch2.peerInfo.addrs) @@ -271,8 +271,6 @@ suite "Network - Test Limits": done = newFuture[void]() switch1 = newStandardSwitch() switch2 = newStandardSwitch() - await switch1.start() - await switch2.start() network1 = BlockExcNetwork.new( switch = switch1, @@ -283,6 +281,9 @@ suite "Network - Test Limits": switch = switch2) switch2.mount(network2) + await switch1.start() + await switch2.start() + await switch1.connect( switch2.peerInfo.peerId, switch2.peerInfo.addrs) diff --git a/tests/codex/helpers.nim b/tests/codex/helpers.nim index 475bf7a1..6d431870 100644 --- a/tests/codex/helpers.nim +++ b/tests/codex/helpers.nim @@ -37,12 +37,11 @@ proc corruptBlocks*( manifest: Manifest, blks, bytes: int): Future[seq[int]] {.async.} = var pos: seq[int] - while true: - if pos.len >= blks: - break - var i = -1 - if (i = Rng.instance.rand(manifest.len - 1); pos.find(i) >= 0): + doAssert blks < manifest.len + while pos.len < blks: + let i = Rng.instance.rand(manifest.len - 1) + if pos.find(i) >= 0: continue pos.add(i) @@ -50,15 +49,12 @@ proc corruptBlocks*( blk = (await store.getBlock(manifest[i])).tryGet() bytePos: seq[int] - while true: - if bytePos.len > bytes: - break - - var ii = -1 - if (ii = Rng.instance.rand(blk.data.len - 1); bytePos.find(ii) >= 0): + doAssert bytes < blk.data.len + while bytePos.len <= bytes: + let ii = Rng.instance.rand(blk.data.len - 1) + if bytePos.find(ii) >= 0: continue bytePos.add(ii) blk.data[ii] = byte 0 - return pos diff --git a/vendor/nim-chronicles b/vendor/nim-chronicles index 16820963..7631f7b2 160000 --- a/vendor/nim-chronicles +++ b/vendor/nim-chronicles @@ -1 +1 @@ -Subproject commit 1682096306ddba8185dcfac360a8c3f952d721e4 +Subproject commit 7631f7b2ee03398cb1512a79923264e8f9410af6 diff --git a/vendor/nim-chronos b/vendor/nim-chronos index 17fed89c..8e826337 160000 --- a/vendor/nim-chronos +++ b/vendor/nim-chronos @@ -1 +1 @@ -Subproject commit 17fed89c99beac5a92d3668d0d3e9b0e4ac13936 +Subproject commit 8e8263370b51bc9b71916273e3eb956053532c4f diff --git a/vendor/nim-http-utils b/vendor/nim-http-utils index 689da19e..e88e231d 160000 --- a/vendor/nim-http-utils +++ b/vendor/nim-http-utils @@ -1 +1 @@ -Subproject commit 689da19e9e9cfff4ced85e2b25c6b2b5598ed079 +Subproject commit e88e231dfcef4585fe3b2fbd9b664dbd28a88040 diff --git a/vendor/nim-json-serialization b/vendor/nim-json-serialization index c5f0e246..e5b18fb7 160000 --- a/vendor/nim-json-serialization +++ b/vendor/nim-json-serialization @@ -1 +1 @@ -Subproject commit c5f0e2465e8375dfc7aa0f56ccef67cb680bc6b0 +Subproject commit e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4 diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index eeb3c210..a3e9d1ed 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit eeb3c210a37408716b6a8b45f578adf87610cef2 +Subproject commit a3e9d1ed80c048cd5abc839cbe0863cefcedc702 diff --git a/vendor/nim-libp2p-dht b/vendor/nim-libp2p-dht index d6d255b4..e4e7a3e1 160000 --- a/vendor/nim-libp2p-dht +++ b/vendor/nim-libp2p-dht @@ -1 +1 @@ -Subproject commit d6d255b4b5d6a4fa56db0eb6677ed7391cbb4897 +Subproject commit e4e7a3e11fe635de3f15e37164b3ace96f588993 diff --git a/vendor/nim-presto b/vendor/nim-presto index 962bb588..3984431d 160000 --- a/vendor/nim-presto +++ b/vendor/nim-presto @@ -1 +1 @@ -Subproject commit 962bb588d19c7180e39f0d9f18131e75861bab20 +Subproject commit 3984431dc0fc829eb668e12e57e90542b041d298 diff --git a/vendor/nim-serialization b/vendor/nim-serialization index fcd0eada..493d18b8 160000 --- a/vendor/nim-serialization +++ b/vendor/nim-serialization @@ -1 +1 @@ -Subproject commit fcd0eadadde0ee000a63df8ab21dc4e9f015a790 +Subproject commit 493d18b8292fc03aa4f835fd825dea1183f97466 diff --git a/vendor/nim-stew b/vendor/nim-stew index 6ad35b87..0c379cf1 160000 --- a/vendor/nim-stew +++ b/vendor/nim-stew @@ -1 +1 @@ -Subproject commit 6ad35b876fb6ebe0dfee0f697af173acc47906ee +Subproject commit 0c379cf1d8d3d9db07af108cc78ff542b2105914 diff --git a/vendor/nim-websock b/vendor/nim-websock index a697e358..7b2ed397 160000 --- a/vendor/nim-websock +++ b/vendor/nim-websock @@ -1 +1 @@ -Subproject commit a697e3585d583ab6b91a159ea7d023461002c927 +Subproject commit 7b2ed397d6e4c37ea4df08ae82aeac7ff04cd180