From 61b18bbe9384ccf4b0a3a779d701d798177b5fac Mon Sep 17 00:00:00 2001 From: "Michael Bradley, Jr" Date: Tue, 13 Sep 2022 19:13:16 -0500 Subject: [PATCH] [build] track nim-libp2p's unstable branch --- .gitignore | 1 + codex/blockexchange/engine/engine.nim | 2 +- codex/blockexchange/engine/pendingblocks.nim | 2 +- codex/erasure/erasure.nim | 16 +++++------ codex/manifest/manifest.nim | 2 +- codex/node.nim | 24 ++++++++--------- codex/rest/api.nim | 2 +- codex/storageproofs/stpnetwork.nim | 2 +- codex/stores/cachestore.nim | 10 +++---- codex/stores/fsstore.nim | 28 ++++++++++---------- codex/stores/networkstore.nim | 10 +++---- codex/streams/storestream.nim | 4 +-- 12 files changed, 52 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index ffb5f8b7..e324f025 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ nimble.paths .update.timestamp codex.nims +nimbus-build-system.paths diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 91a3e8d4..f8343a7f 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -302,7 +302,7 @@ proc blocksHandler*( trace "Got blocks from peer", peer, len = blocks.len for blk in blocks: if isErr (await b.localStore.putBlock(blk)): - trace "Unable to store block", cid = blk.cid + trace "Unable to store block", cid = $blk.cid await b.resolveBlocks(blocks) let diff --git a/codex/blockexchange/engine/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim index dd24b1d5..ea379415 100644 --- a/codex/blockexchange/engine/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -52,7 +52,7 @@ proc getWantHandle*( return await p.blocks[cid].handle.wait(timeout) except CancelledError as exc: - trace "Blocks cancelled", exc = exc.msg, cid + trace "Blocks cancelled", exc = exc.msg, cid = $cid raise exc except CatchableError as exc: trace "Pending WANT failed or expired", exc = exc.msg diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index ccc793d5..a5ca9b80 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -76,7 +76,7 @@ proc encode*( ## logScope: - original_cid = manifest.cid.get() + original_cid = $manifest.cid.get() original_len = manifest.len blocks = blocks parity = parity @@ -118,7 +118,7 @@ proc encode*( trace "Unable to retrieve block", error = error.msg return failure error - trace "Encoding block", cid = blk.cid, pos = idx + trace "Encoding block", cid = $blk.cid, pos = idx shallowCopy(data[j], blk.data) else: trace "Padding with empty block", pos = idx @@ -137,10 +137,10 @@ proc encode*( trace "Unable to create parity block", err = error.msg return failure(error) - trace "Adding parity block", cid = blk.cid, pos = idx + trace "Adding parity block", cid = $blk.cid, pos = idx encoded[idx] = blk.cid if isErr (await self.store.putBlock(blk)): - trace "Unable to store block!", cid = blk.cid + trace "Unable to store block!", cid = $blk.cid return failure("Unable to store block!") except CancelledError as exc: trace "Erasure coding encoding cancelled" @@ -212,10 +212,10 @@ proc decode*( continue if idx >= encoded.K: - trace "Retrieved parity block", cid = blk.cid, idx + trace "Retrieved parity block", cid = $blk.cid, idx shallowCopy(parityData[idx - encoded.K], if blk.isEmpty: emptyBlock else: blk.data) else: - trace "Retrieved data block", cid = blk.cid, idx + trace "Retrieved data block", cid = $blk.cid, idx shallowCopy(data[idx], if blk.isEmpty: emptyBlock else: blk.data) resolved.inc @@ -241,9 +241,9 @@ proc decode*( trace "Unable to create block!", exc = error.msg return failure(error) - trace "Recovered block", cid = blk.cid + trace "Recovered block", cid = $blk.cid if isErr (await self.store.putBlock(blk)): - trace "Unable to store block!", cid = blk.cid + trace "Unable to store block!", cid = $blk.cid return failure("Unable to store block!") except CancelledError as exc: trace "Erasure coding decoding cancelled" diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 8be8d4bb..0af000b5 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -49,7 +49,7 @@ func `[]=`*(self: Manifest, i: BackwardsIndex, item: Cid) = proc add*(self: Manifest, cid: Cid) = assert not self.protected # we expect that protected manifests are created with properly-sized self.blocks self.rootHash = Cid.none - trace "Adding cid to manifest", cid + trace "Adding cid to manifest", cid = $cid self.blocks.add(cid) self.originalBytes = self.blocks.len * self.blockSize diff --git a/codex/node.nim b/codex/node.nim index 177489a9..ca1a3218 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -73,7 +73,7 @@ proc fetchManifest*( containerType =? ManifestContainers.?[$contentType]: return failure "CID has invalid content type for manifest" - trace "Received retrieval request", cid + trace "Received retrieval request", cid = $cid without blk =? await node.blockStore.getBlock(cid), error: return failure error @@ -126,9 +126,9 @@ proc retrieve*( try: # Spawn an erasure decoding job without res =? (await node.erasure.decode(manifest)), error: - trace "Unable to erasure decode manifest", cid, exc = error.msg + trace "Unable to erasure decode manifest", cid = $cid, exc = error.msg except CatchableError as exc: - trace "Exception decoding manifest", cid + trace "Exception decoding manifest", cid = $cid # asyncSpawn erasureJob() else: @@ -219,8 +219,8 @@ proc store*( trace "Unable to generate manifest Cid!", exc = error.msg return failure(error.msg) - trace "Stored data", manifestCid = manifest.cid, - contentCid = cid, + trace "Stored data", manifestCid = $manifest.cid, + contentCid = $cid, blocks = blockManifest.len # Announce manifest @@ -244,19 +244,19 @@ proc requestStorage*(self: CodexNodeRef, ## - Run the PoR setup on the erasure dataset ## - Call into the marketplace and purchasing contracts ## - trace "Received a request for storage!", cid, duration, nodes, tolerance, reward + trace "Received a request for storage!", cid = $cid, duration, nodes, tolerance, reward without contracts =? self.contracts: trace "Purchasing not available" return failure "Purchasing not available" without manifest =? await self.fetchManifest(cid), error: - trace "Unable to fetch manifest for cid", cid + trace "Unable to fetch manifest for cid", cid = $cid raise error # Erasure code the dataset according to provided parameters without encoded =? (await self.erasure.encode(manifest, nodes.int, tolerance.int)), error: - trace "Unable to erasure code dataset", cid + trace "Unable to erasure code dataset", cid = $cid return failure(error) without encodedData =? encoded.encode(), error: @@ -268,7 +268,7 @@ proc requestStorage*(self: CodexNodeRef, return failure(error) if isErr (await self.blockStore.putBlock(encodedBlk)): - trace "Unable to store encoded manifest block", cid = encodedBlk.cid + trace "Unable to store encoded manifest block", cid = $encodedBlk.cid return failure("Unable to store encoded manifest block") let request = StorageRequest( @@ -334,14 +334,14 @@ proc start*(node: CodexNodeRef) {.async.} = ## without cid =? Cid.init(request.content.cid): - trace "Unable to parse Cid", cid + trace "Unable to parse Cid", cid = $cid raise newException(CodexError, "Unable to parse Cid") without manifest =? await node.fetchManifest(cid), error: - trace "Unable to fetch manifest for cid", cid + trace "Unable to fetch manifest for cid", cid = $cid raise error - trace "Fetching block for manifest", cid + trace "Fetching block for manifest", cid = $cid # TODO: This will probably require a call to `getBlock` either way, # since fetching of blocks will have to be selective according # to a combination of parameters, such as node slot position diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 0fd87b43..d42971db 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -123,7 +123,7 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = trace "Excepting streaming blocks", exc = exc.msg return RestApiResponse.error(Http500) finally: - trace "Sent bytes", cid = id.get(), bytes + trace "Sent bytes", cid = $id.get(), bytes if not stream.isNil: await stream.close() diff --git a/codex/storageproofs/stpnetwork.nim b/codex/storageproofs/stpnetwork.nim index 99abd990..bd77034a 100644 --- a/codex/storageproofs/stpnetwork.nim +++ b/codex/storageproofs/stpnetwork.nim @@ -63,7 +63,7 @@ proc uploadTags*( except CancelledError as exc: raise exc except CatchableError as exc: - trace "Exception submitting tags", cid, exc = exc.msg + trace "Exception submitting tags", cid = $cid, exc = exc.msg return failure(exc.msg) finally: await conn.close() diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index aa8eeeb7..af8c006a 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -47,7 +47,7 @@ method getBlock*(self: CacheStore, cid: Cid): Future[?!Block] {.async.} = ## Get a block from the stores ## - trace "Getting block from cache", cid + trace "Getting block from cache", cid = $cid if cid.isEmpty: trace "Empty block, ignoring" @@ -59,14 +59,14 @@ method getBlock*(self: CacheStore, cid: Cid): Future[?!Block] {.async.} = try: return success self.cache[cid] except CatchableError as exc: - trace "Error requesting block from cache", cid, error = exc.msg + trace "Error requesting block from cache", cid = $cid, error = exc.msg return failure exc method hasBlock*(self: CacheStore, cid: Cid): Future[?!bool] {.async.} = ## Check if the block exists in the blockstore ## - trace "Checking CacheStore for block presence", cid + trace "Checking CacheStore for block presence", cid = $cid if cid.isEmpty: trace "Empty block, ignoring" return true.success @@ -108,7 +108,7 @@ method putBlock*(self: CacheStore, blk: Block): Future[?!void] {.async.} = ## Put a block to the blockstore ## - trace "Storing block in cache", cid = blk.cid + trace "Storing block in cache", cid = $blk.cid if blk.isEmpty: trace "Empty block, ignoring" return success() @@ -120,7 +120,7 @@ method delBlock*(self: CacheStore, cid: Cid): Future[?!void] {.async.} = ## Delete a block from the blockstore ## - trace "Deleting block from cache", cid + trace "Deleting block from cache", cid = $cid if cid.isEmpty: trace "Empty block, ignoring" return success() diff --git a/codex/stores/fsstore.nim b/codex/stores/fsstore.nim index 10af63bd..7d0cc1d5 100644 --- a/codex/stores/fsstore.nim +++ b/codex/stores/fsstore.nim @@ -43,9 +43,9 @@ method getBlock*(self: FSStore, cid: Cid): Future[?!Block] {.async.} = ## if not self.cache.isNil: - trace "Getting block from cache or filestore", cid + trace "Getting block from cache or filestore", cid = $cid else: - trace "Getting block from filestore", cid + trace "Getting block from filestore", cid = $cid if cid.isEmpty: trace "Empty block, ignoring" @@ -58,7 +58,7 @@ method getBlock*(self: FSStore, cid: Cid): Future[?!Block] {.async.} = if not cachedBlockRes.isErr: return success cachedBlockRes.get else: - trace "Unable to read block from cache", cid, error = cachedBlockRes.error.msg + trace "Unable to read block from cache", cid = $cid, error = cachedBlockRes.error.msg # Read file contents var @@ -79,7 +79,7 @@ method getBlock*(self: FSStore, cid: Cid): Future[?!Block] {.async.} = return failure "Error requesting block from filestore: " & error without blk =? Block.new(cid, data), error: - trace "Unable to construct block from data", cid, error = error.msg + trace "Unable to construct block from data", cid = $cid, error = error.msg return failure error if not self.cache.isNil: @@ -87,7 +87,7 @@ method getBlock*(self: FSStore, cid: Cid): Future[?!Block] {.async.} = putCachedRes = await self.cache.putBlock(blk) if putCachedRes.isErr: - trace "Unable to store block in cache", cid, error = putCachedRes.error.msg + trace "Unable to store block in cache", cid = $cid, error = putCachedRes.error.msg return success blk @@ -97,9 +97,9 @@ method putBlock*(self: FSStore, blk: Block): Future[?!void] {.async.} = ## if not self.cache.isNil: - trace "Putting block into filestore and cache", cid = blk.cid + trace "Putting block into filestore and cache", cid = $blk.cid else: - trace "Putting block into filestore", cid = blk.cid + trace "Putting block into filestore", cid = $blk.cid if blk.isEmpty: trace "Empty block, ignoring" @@ -118,7 +118,7 @@ method putBlock*(self: FSStore, blk: Block): Future[?!void] {.async.} = let res = io2.writeFile(path, blk.data) if res.isErr: let error = io2.ioErrorMsg(res.error) - trace "Unable to store block", path, cid = blk.cid, error + trace "Unable to store block", path, cid = $blk.cid, error return failure("Unable to store block") if not self.cache.isNil: @@ -126,7 +126,7 @@ method putBlock*(self: FSStore, blk: Block): Future[?!void] {.async.} = putCachedRes = await self.cache.putBlock(blk) if putCachedRes.isErr: - trace "Unable to store block in cache", cid = blk.cid, error = putCachedRes.error.msg + trace "Unable to store block in cache", cid = $blk.cid, error = putCachedRes.error.msg return success() @@ -135,9 +135,9 @@ method delBlock*(self: FSStore, cid: Cid): Future[?!void] {.async.} = ## if not self.cache.isNil: - trace "Deleting block from cache and filestore", cid + trace "Deleting block from cache and filestore", cid = $cid else: - trace "Deleting block from filestore", cid + trace "Deleting block from filestore", cid = $cid if cid.isEmpty: trace "Empty block, ignoring" @@ -148,7 +148,7 @@ method delBlock*(self: FSStore, cid: Cid): Future[?!void] {.async.} = delCachedRes = await self.cache.delBlock(cid) if delCachedRes.isErr: - trace "Unable to delete block from cache", cid, error = delCachedRes.error.msg + trace "Unable to delete block from cache", cid = $cid, error = delCachedRes.error.msg let path = self.blockPath(cid) @@ -156,7 +156,7 @@ method delBlock*(self: FSStore, cid: Cid): Future[?!void] {.async.} = if res.isErr: let error = io2.ioErrorMsg(res.error) - trace "Unable to delete block", path, cid, error + trace "Unable to delete block", path, cid = $cid, error return error.failure return success() @@ -165,7 +165,7 @@ method hasBlock*(self: FSStore, cid: Cid): Future[?!bool] {.async.} = ## Check if a block exists in the filestore ## - trace "Checking filestore for block existence", cid + trace "Checking filestore for block existence", cid = $cid if cid.isEmpty: trace "Empty block, ignoring" return true.success diff --git a/codex/stores/networkstore.nim b/codex/stores/networkstore.nim index 5c96a5f6..730141c0 100644 --- a/codex/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -35,11 +35,11 @@ method getBlock*(self: NetworkStore, cid: Cid): Future[?!bt.Block] {.async.} = ## Get a block from a remote peer ## - trace "Getting block from local store or network", cid + trace "Getting block from local store or network", cid = $cid without blk =? await self.localStore.getBlock(cid), error: if not (error of BlockNotFoundError): return failure error - trace "Block not in local store", cid + trace "Block not in local store", cid = $cid # TODO: What if block isn't available in the engine too? # TODO: add retrieved block to the local store return (await self.engine.requestBlock(cid)).catch @@ -50,7 +50,7 @@ method putBlock*(self: NetworkStore, blk: bt.Block): Future[?!void] {.async.} = ## Store block locally and notify the network ## - trace "Puting block into network store", cid = blk.cid + trace "Puting block into network store", cid = $blk.cid let res = await self.localStore.putBlock(blk) if res.isErr: @@ -63,7 +63,7 @@ method delBlock*(self: NetworkStore, cid: Cid): Future[?!void] = ## Delete a block from the blockstore ## - trace "Deleting block from network store", cid + trace "Deleting block from network store", cid = $cid return self.localStore.delBlock(cid) {.pop.} @@ -72,7 +72,7 @@ method hasBlock*(self: NetworkStore, cid: Cid): Future[?!bool] {.async.} = ## Check if the block exists in the blockstore ## - trace "Checking network store for block existence", cid + trace "Checking network store for block existence", cid = $cid return await self.localStore.hasBlock(cid) method close*(self: NetworkStore): Future[void] {.async.} = diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index 6649d1bc..37039ea7 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -69,7 +69,7 @@ method readOnce*( ## Return how many bytes were actually read before EOF was encountered. ## Raise exception if we are already at EOF. - trace "Reading from manifest", cid = self.manifest.cid.get(), blocks = self.manifest.len + trace "Reading from manifest", cid = $self.manifest.cid.get(), blocks = self.manifest.len if self.atEof: raise newLPStreamEOFError() @@ -88,7 +88,7 @@ method readOnce*( without blk =? await self.store.getBlock(self.manifest[blockNum]), error: raise newLPStreamReadError(error) - trace "Reading bytes from store stream", blockNum, cid = blk.cid, bytes = readBytes, blockOffset + trace "Reading bytes from store stream", blockNum, cid = $blk.cid, bytes = readBytes, blockOffset # Copy `readBytes` bytes starting at `blockOffset` from the block into the outbuf if blk.isEmpty: