From 0107eb06fe61ec0a86b8f60ae58d0833ccf0c2d2 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Tue, 18 Feb 2025 06:47:47 +0100 Subject: [PATCH] chore(marketplace): cid should be bytes (#1073) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Change cid format from string to bytes for the marketplace * refactor: marketplace custom errors handling * chore: update contracts repo * chore: update contracts submodule * Update contracts submodule * Initialize the Cid using init function * Restorage serialize pragma * Use Cid object instead of buffer * Simplify cid usage * Simplify cid usage * bump codex-contracts-eth after PR merge, formatting * fix rebase * collateralPerByte => collateralPerSlot --------- Co-authored-by: Adam Uhlíř Co-authored-by: Eric <5089238+emizzle@users.noreply.github.com> --- codex/contracts/requests.nim | 19 +++++++++++++++--- codex/node.nim | 26 ++++++++----------------- codex/sales/salescontext.nim | 3 ++- tests/codex/node/testcontracts.nim | 5 ++--- tests/codex/node/testnode.nim | 2 +- tests/codex/sales/states/testfilled.nim | 2 +- tests/codex/sales/testsales.nim | 12 ++++++++---- tests/contracts/testMarket.nim | 1 + tests/examples.nim | 2 +- tests/integration/testecbug.nim | 2 +- tests/integration/testpurchasing.nim | 2 ++ vendor/codex-contracts-eth | 2 +- 12 files changed, 44 insertions(+), 34 deletions(-) diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 48947602..98d8c342 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -6,8 +6,10 @@ import pkg/nimcrypto import pkg/ethers/fields import pkg/questionable/results import pkg/stew/byteutils +import pkg/libp2p/[cid, multicodec] import ../logutils import ../utils/json +from ../errors import mapFailure export contractabi @@ -29,7 +31,7 @@ type maxSlotLoss* {.serialize.}: uint64 StorageContent* = object - cid* {.serialize.}: string + cid* {.serialize.}: Cid merkleRoot*: array[32, byte] Slot* = object @@ -120,6 +122,9 @@ func fromTuple(_: type StorageAsk, tupl: tuple): StorageAsk = func fromTuple(_: type StorageContent, tupl: tuple): StorageContent = StorageContent(cid: tupl[0], merkleRoot: tupl[1]) +func solidityType*(_: type Cid): string = + solidityType(seq[byte]) + func solidityType*(_: type StorageContent): string = solidityType(StorageContent.fieldTypes) @@ -129,6 +134,10 @@ func solidityType*(_: type StorageAsk): string = func solidityType*(_: type StorageRequest): string = solidityType(StorageRequest.fieldTypes) +# Note: it seems to be ok to ignore the vbuffer offset for now +func encode*(encoder: var AbiEncoder, cid: Cid) = + encoder.write(cid.data.buffer) + func encode*(encoder: var AbiEncoder, content: StorageContent) = encoder.write(content.fieldValues) @@ -141,8 +150,12 @@ func encode*(encoder: var AbiEncoder, id: RequestId | SlotId | Nonce) = func encode*(encoder: var AbiEncoder, request: StorageRequest) = encoder.write(request.fieldValues) -func encode*(encoder: var AbiEncoder, request: Slot) = - encoder.write(request.fieldValues) +func encode*(encoder: var AbiEncoder, slot: Slot) = + encoder.write(slot.fieldValues) + +func decode*(decoder: var AbiDecoder, T: type Cid): ?!T = + let data = ?decoder.read(seq[byte]) + Cid.init(data).mapFailure func decode*(decoder: var AbiDecoder, T: type StorageContent): ?!T = let tupl = ?decoder.read(StorageContent.fieldTypes) diff --git a/codex/node.nim b/codex/node.nim index 2602bfe6..062ec2ce 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -501,10 +501,7 @@ proc setupRequest( collateralPerByte: collateralPerByte, maxSlotLoss: tolerance, ), - content: StorageContent( - cid: $manifestBlk.cid, # TODO: why string? - merkleRoot: verifyRoot, - ), + content: StorageContent(cid: manifestBlk.cid, merkleRoot: verifyRoot), expiry: expiry, ) @@ -561,16 +558,14 @@ proc onStore( ## store data in local storage ## + let cid = request.content.cid + logScope: - cid = request.content.cid + cid = $cid slotIdx = slotIdx trace "Received a request to store a slot" - without cid =? Cid.init(request.content.cid).mapFailure, err: - trace "Unable to parse Cid", cid - return failure(err) - without manifest =? (await self.fetchManifest(cid)), err: trace "Unable to fetch manifest for cid", cid, err = err.msg return failure(err) @@ -640,7 +635,7 @@ proc onProve( ## let - cidStr = slot.request.content.cid + cidStr = $slot.request.content.cid slotIdx = slot.slotIndex.truncate(Natural) logScope: @@ -689,14 +684,9 @@ proc onProve( failure "Prover not enabled" proc onExpiryUpdate( - self: CodexNodeRef, rootCid: string, expiry: SecondsSince1970 + self: CodexNodeRef, rootCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async.} = - without cid =? Cid.init(rootCid): - trace "Unable to parse Cid", cid - let error = newException(CodexError, "Unable to parse Cid") - return failure(error) - - return await self.updateExpiry(cid, expiry) + return await self.updateExpiry(rootCid, expiry) proc onClear(self: CodexNodeRef, request: StorageRequest, slotIndex: UInt256) = # TODO: remove data from local storage @@ -719,7 +709,7 @@ proc start*(self: CodexNodeRef) {.async.} = self.onStore(request, slot, onBatch) hostContracts.sales.onExpiryUpdate = proc( - rootCid: string, expiry: SecondsSince1970 + rootCid: Cid, expiry: SecondsSince1970 ): Future[?!void] = self.onExpiryUpdate(rootCid, expiry) diff --git a/codex/sales/salescontext.nim b/codex/sales/salescontext.nim index bb0b5dc9..95f06c04 100644 --- a/codex/sales/salescontext.nim +++ b/codex/sales/salescontext.nim @@ -1,6 +1,7 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises +import pkg/libp2p/cid import ../market import ../clock @@ -30,7 +31,7 @@ type OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[?!Groth16Proof] {. gcsafe, upraises: [] .} - OnExpiryUpdate* = proc(rootCid: string, expiry: SecondsSince1970): Future[?!void] {. + OnExpiryUpdate* = proc(rootCid: Cid, expiry: SecondsSince1970): Future[?!void] {. gcsafe, upraises: [] .} OnClear* = proc(request: StorageRequest, slotIndex: UInt256) {.gcsafe, upraises: [].} diff --git a/tests/codex/node/testcontracts.nim b/tests/codex/node/testcontracts.nim index 52adb5f6..0930d925 100644 --- a/tests/codex/node/testcontracts.nim +++ b/tests/codex/node/testcontracts.nim @@ -78,7 +78,6 @@ asyncchecksuite "Test Node - Host contracts": erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, Taskpool.new) manifestCid = manifestBlock.cid - manifestCidStr = $(manifestCid) (await localStore.putBlock(manifestBlock)).tryGet() @@ -99,7 +98,7 @@ asyncchecksuite "Test Node - Host contracts": expectedExpiry: SecondsSince1970 = clock.now + DefaultBlockTtl.seconds + 11123 expiryUpdateCallback = !sales.onExpiryUpdate - (await expiryUpdateCallback(manifestCidStr, expectedExpiry)).tryGet() + (await expiryUpdateCallback(manifestCid, expectedExpiry)).tryGet() for index in 0 ..< manifest.blocksCount: let @@ -116,7 +115,7 @@ asyncchecksuite "Test Node - Host contracts": test "onStore callback": let onStore = !sales.onStore var request = StorageRequest.example - request.content.cid = $verifiableBlock.cid + request.content.cid = verifiableBlock.cid request.expiry = (getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix.u256 var fetchedBytes: uint = 0 diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index 3f9a141a..37960232 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -167,7 +167,7 @@ asyncchecksuite "Test Node - Basic": check: (await verifiableBlock.cid in localStore) == true - request.content.cid == $verifiableBlock.cid + request.content.cid == verifiableBlock.cid request.content.merkleRoot == builder.verifyRoot.get.toBytes test "Should delete a single block": diff --git a/tests/codex/sales/states/testfilled.nim b/tests/codex/sales/states/testfilled.nim index f8f77da6..74413776 100644 --- a/tests/codex/sales/states/testfilled.nim +++ b/tests/codex/sales/states/testfilled.nim @@ -36,7 +36,7 @@ checksuite "sales state 'filled'": market.requestEnds[request.id] = 321 onExpiryUpdatePassedExpiry = -1 let onExpiryUpdate = proc( - rootCid: string, expiry: SecondsSince1970 + rootCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async.} = onExpiryUpdatePassedExpiry = expiry return success() diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim index 0d441f34..05f31057 100644 --- a/tests/codex/sales/testsales.nim +++ b/tests/codex/sales/testsales.nim @@ -46,7 +46,9 @@ asyncchecksuite "Sales - start": pricePerBytePerSecond: 1.u256, collateralPerByte: 1.u256, ), - content: StorageContent(cid: "some cid"), + content: StorageContent( + cid: Cid.init("zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob").tryGet + ), expiry: (getTime() + initDuration(hours = 1)).toUnix.u256, ) @@ -64,7 +66,7 @@ asyncchecksuite "Sales - start": return success() sales.onExpiryUpdate = proc( - rootCid: string, expiry: SecondsSince1970 + rootCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async.} = return success() @@ -158,7 +160,9 @@ asyncchecksuite "Sales": pricePerBytePerSecond: minPricePerBytePerSecond, collateralPerByte: 1.u256, ), - content: StorageContent(cid: "some cid"), + content: StorageContent( + cid: Cid.init("zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob").tryGet + ), expiry: (getTime() + initDuration(hours = 1)).toUnix.u256, ) @@ -181,7 +185,7 @@ asyncchecksuite "Sales": return success() sales.onExpiryUpdate = proc( - rootCid: string, expiry: SecondsSince1970 + rootCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async.} = return success() diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 2ba450a1..6506a2d6 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -3,6 +3,7 @@ import std/importutils import pkg/chronos import pkg/ethers/erc20 import codex/contracts +import pkg/libp2p/cid import pkg/lrucache import ../ethertest import ./examples diff --git a/tests/examples.nim b/tests/examples.nim index c96fefd6..26013cdc 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -57,7 +57,7 @@ proc example*(_: type StorageRequest): StorageRequest = maxSlotLoss: 2, # 2 slots can be freed without data considered to be lost ), content: StorageContent( - cid: "zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob", + cid: Cid.init("zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob").tryGet, merkleRoot: array[32, byte].example, ), expiry: (60 * 60).u256, # 1 hour , diff --git a/tests/integration/testecbug.nim b/tests/integration/testecbug.nim index e7604de7..29a3bc6f 100644 --- a/tests/integration/testecbug.nim +++ b/tests/integration/testecbug.nim @@ -50,7 +50,7 @@ marketplacesuite "Bug #821 - node crashes during erasure coding": check eventually(requestId.isSome, timeout = expiry.int * 1000) let request = await marketplace.getRequest(requestId.get) - let cidFromRequest = Cid.init(request.content.cid).get() + let cidFromRequest = request.content.cid let downloaded = await clientApi.downloadBytes(cidFromRequest, local = true) check downloaded.isOk check downloaded.get.toHex == data.toHex diff --git a/tests/integration/testpurchasing.nim b/tests/integration/testpurchasing.nim index 4e08e7a8..ebae78f6 100644 --- a/tests/integration/testpurchasing.nim +++ b/tests/integration/testpurchasing.nim @@ -47,6 +47,8 @@ twonodessuite "Purchasing": ).get let request = client1.getPurchase(id).get.request.get + + check request.content.cid.data.buffer.len > 0 check request.ask.duration == 100.u256 check request.ask.pricePerBytePerSecond == 1.u256 check request.ask.proofProbability == 3.u256 diff --git a/vendor/codex-contracts-eth b/vendor/codex-contracts-eth index e74d3397..0f2012b1 160000 --- a/vendor/codex-contracts-eth +++ b/vendor/codex-contracts-eth @@ -1 +1 @@ -Subproject commit e74d3397a133eaf1eb95d9ce59f56747a7c8c30b +Subproject commit 0f2012b1442c404605c8ba9dcae2f4e53058cd2c