diff --git a/.github/workflows/ci-reusable.yml b/.github/workflows/ci-reusable.yml index b6131b18..93b47d1d 100644 --- a/.github/workflows/ci-reusable.yml +++ b/.github/workflows/ci-reusable.yml @@ -48,29 +48,7 @@ jobs: if: matrix.tests == 'unittest' || matrix.tests == 'all' run: make -j${ncpu} test - - name: Setup Node.js - if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all' - uses: actions/setup-node@v4 - with: - node-version: 22 - - - name: Start Ethereum node with Logos Storage contracts - if: matrix.tests == 'contract' || matrix.tests == 'integration' || matrix.tests == 'tools' || matrix.tests == 'all' - working-directory: vendor/logos-storage-contracts-eth - env: - MSYS2_PATH_TYPE: inherit - run: | - npm ci - npm start & - # Wait for the contracts to be deployed - sleep 5 - ## Part 2 Tests ## - - name: Contract tests - if: matrix.tests == 'contract' || matrix.tests == 'all' - run: make -j${ncpu} testContracts - - ## Part 3 Tests ## - name: Integration tests if: matrix.tests == 'integration' || matrix.tests == 'all' env: @@ -85,11 +63,6 @@ jobs: path: tests/integration/logs/ retention-days: 1 - ## Part 4 Tools ## - - name: Tools tests - if: matrix.tests == 'tools' || matrix.tests == 'all' - run: make -j${ncpu} testTools - status: if: always() needs: [build] diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2b62e3b2..6dd4abd6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,26 +19,10 @@ on: workflow_dispatch: jobs: - get-contracts-hash: - runs-on: ubuntu-latest - outputs: - hash: ${{ steps.get-hash.outputs.hash }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Get submodule short hash - id: get-hash - run: | - hash=$(git rev-parse --short HEAD:vendor/logos-storage-contracts-eth) - echo "hash=$hash" >> $GITHUB_OUTPUT build-and-push: name: Build and Push uses: ./.github/workflows/docker-reusable.yml - needs: get-contracts-hash with: tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} tag_stable: ${{ startsWith(github.ref, 'refs/tags/') }} - contract_image: "codexstorage/codex-contracts-eth:sha-${{ needs.get-contracts-hash.outputs.hash }}" secrets: inherit \ No newline at end of file diff --git a/Makefile b/Makefile index d9b9d70e..380814a4 100644 --- a/Makefile +++ b/Makefile @@ -98,11 +98,6 @@ all: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim storage $(NIM_PARAMS) build.nims -# Build tools/cirdl -cirdl: | deps - echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim toolsCirdl $(NIM_PARAMS) build.nims - # must be included after the default target -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk @@ -135,11 +130,6 @@ test: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim test $(NIM_PARAMS) build.nims -# Builds and runs the smart contract tests -testContracts: | build deps - echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testContracts $(NIM_PARAMS) --define:ws_resubscribe=240 build.nims - # Builds and runs the integration tests testIntegration: | build deps echo -e $(BUILD_MSG) "build/$@" && \ @@ -150,16 +140,6 @@ testAll: | build deps echo -e $(BUILD_MSG) "build/$@" && \ $(ENV_SCRIPT) nim testAll $(NIM_PARAMS) build.nims -# Builds and runs Taiko L2 tests -testTaiko: | build deps - echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testTaiko $(NIM_PARAMS) build.nims - -# Builds and runs tool tests -testTools: | cirdl - echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim testTools $(NIM_PARAMS) build.nims - # nim-libbacktrace LIBBACKTRACE_MAKE_FLAGS := -C vendor/nim-libbacktrace --no-print-directory BUILD_CXX_LIB=0 libbacktrace: diff --git a/build.nims b/build.nims index 47e848b3..d9dac119 100644 --- a/build.nims +++ b/build.nims @@ -3,7 +3,7 @@ mode = ScriptMode.Verbose import std/os except commandLineParams ### Helper functions -proc buildBinary(srcName: string, outName = os.lastPathPart(srcName), srcDir = "./", params = "", lang = "c") = +proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") = if not dirExists "build": mkDir "build" @@ -18,9 +18,10 @@ proc buildBinary(srcName: string, outName = os.lastPathPart(srcName), srcDir = let # Place build output in 'build' folder, even if name includes a longer path. + outName = os.lastPathPart(name) cmd = "nim " & lang & " --out:build/" & outName & " " & extra_params & " " & srcDir & - srcName & ".nim" + name & ".nim" exec(cmd) @@ -36,65 +37,50 @@ proc buildLibrary(name: string, srcDir = "./", params = "", `type` = "dynamic") ) exec "nim c" & " --out:build/" & lib_name & " --threads:on --app:lib --opt:size --noMain --mm:refc --header --d:metrics " & - "--nimMainPrefix:libstorage -d:noSignalHandler " & + "--nimMainPrefix:libcodex -d:noSignalHandler " & "-d:LeopardExtraCompilerFlags=-fPIC " & "-d:chronicles_runtime_filtering " & "-d:chronicles_log_level=TRACE " & params & " " & srcDir & name & ".nim" else: exec "nim c" & " --out:build/" & name & ".a --threads:on --app:staticlib --opt:size --noMain --mm:refc --header --d:metrics " & - "--nimMainPrefix:libstorage -d:noSignalHandler " & + "--nimMainPrefix:libcodex -d:noSignalHandler " & "-d:LeopardExtraCompilerFlags=-fPIC " & "-d:chronicles_runtime_filtering " & "-d:chronicles_log_level=TRACE " & params & " " & srcDir & name & ".nim" -proc test(name: string, outName = name, srcDir = "tests/", params = "", lang = "c") = - buildBinary name, outName, srcDir, params - exec "build/" & outName +proc test(name: string, srcDir = "tests/", params = "", lang = "c") = + buildBinary name, srcDir, params + exec "build/" & name -task storage, "build logos storage binary": +task codex, "build codex binary": buildBinary "codex", - outname = "storage", params = "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" task toolsCirdl, "build tools/cirdl binary": buildBinary "tools/cirdl/cirdl" task testStorage, "Build & run Logos Storage tests": - test "testCodex", outName = "testStorage", params = "-d:storage_enable_proof_failures=true" - -task testContracts, "Build & run Logos Storage Contract tests": - test "testContracts" + test "testCodex", outName = "testStorage" task testIntegration, "Run integration tests": buildBinary "codex", - outName = "storage", params = - "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE -d:storage_enable_proof_failures=true" + "-d:chronicles_runtime_filtering -d:chronicles_log_level=TRACE" test "testIntegration" # use params to enable logging from the integration test executable # test "testIntegration", params = "-d:chronicles_sinks=textlines[notimestamps,stdout],textlines[dynamic] " & # "-d:chronicles_enabled_topics:integration:TRACE" -task build, "build Logos Storage binary": - storageTask() +task build, "build codex binary": + codexTask() task test, "Run tests": - testStorageTask() - -task testTools, "Run Tools tests": - toolsCirdlTask() - test "testTools" + testCodexTask() task testAll, "Run all tests (except for Taiko L2 tests)": testStorageTask() - testContractsTask() testIntegrationTask() - testToolsTask() - -task testTaiko, "Run Taiko L2 tests": - storageTask() - test "testTaiko" import strutils import os @@ -126,7 +112,7 @@ task coverage, "generates code coverage report": test "coverage", srcDir = "tests/", params = - " --nimcache:nimcache/coverage -d:release -d:storage_enable_proof_failures=true" + " --nimcache:nimcache/coverage -d:release" exec("rm nimcache/coverage/*.c") rmDir("coverage") mkDir("coverage") @@ -147,22 +133,22 @@ task showCoverage, "open coverage html": if findExe("open") != "": exec("open coverage/report/index.html") -task libstorageDynamic, "Generate bindings": +task libcodexDynamic, "Generate bindings": var params = "" when compiles(commandLineParams): for param in commandLineParams(): if param.len > 0 and param.startsWith("-"): params.add " " & param - let name = "libstorage" + let name = "libcodex" buildLibrary name, "library/", params, "dynamic" -task libstorageStatic, "Generate bindings": +task libcodexStatic, "Generate bindings": var params = "" when compiles(commandLineParams): for param in commandLineParams(): if param.len > 0 and param.startsWith("-"): params.add " " & param - let name = "libstorage" + let name = "libcodex" buildLibrary name, "library/", params, "static" diff --git a/codex.nim b/codex.nim index 24d27959..aad998b0 100644 --- a/codex.nim +++ b/codex.nim @@ -71,9 +71,6 @@ when isMainModule: # permissions are insecure. quit QuitFailure - if config.prover() and not (checkAndCreateDataDir((config.circuitDir).string)): - quit QuitFailure - trace "Data dir initialized", dir = $config.dataDir if not (checkAndCreateDataDir((config.dataDir / "repo"))): diff --git a/codex/blockexchange/engine.nim b/codex/blockexchange/engine.nim index 5aeb96ba..b768e8d5 100644 --- a/codex/blockexchange/engine.nim +++ b/codex/blockexchange/engine.nim @@ -1,6 +1,5 @@ import ./engine/discovery import ./engine/advertiser import ./engine/engine -import ./engine/payments -export discovery, advertiser, engine, payments +export discovery, advertiser, engine diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index f9245885..41b76812 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -37,12 +37,11 @@ import ../protobuf/presence import ../network import ../peers -import ./payments import ./discovery import ./advertiser import ./pendingblocks -export peers, pendingblocks, payments, discovery +export peers, pendingblocks, discovery logScope: topics = "codex blockexcengine" @@ -113,16 +112,10 @@ type maxBlocksPerMessage: int # Maximum number of blocks we can squeeze in a single message pendingBlocks*: PendingBlocksManager # Blocks we're awaiting to be resolved - wallet*: WalletRef # Nitro wallet for micropayments - pricing*: ?Pricing # Optional bandwidth pricing discovery*: DiscoveryEngine advertiser*: Advertiser lastDiscRequest: Moment # time of last discovery request - Pricing* = object - address*: EthAddress - price*: UInt256 - # attach task scheduler to engine proc scheduleTask(self: BlockExcEngine, task: BlockExcPeerCtx) {.gcsafe, raises: [].} = if self.taskQueue.pushOrUpdateNoWait(task).isOk(): @@ -644,17 +637,6 @@ proc resolveBlocks*( ) ) -proc payForBlocks( - self: BlockExcEngine, peer: BlockExcPeerCtx, blocksDelivery: seq[BlockDelivery] -) {.async: (raises: [CancelledError]).} = - let - sendPayment = self.network.request.sendPayment - price = peer.price(blocksDelivery.mapIt(it.address)) - - if payment =? self.wallet.pay(peer, price): - trace "Sending payment for blocks", price, len = blocksDelivery.len - await sendPayment(peer.id, payment) - proc validateBlockDelivery(self: BlockExcEngine, bd: BlockDelivery): ?!void = if bd.address notin self.pendingBlocks: return failure("Received block is not currently a pending block") @@ -749,11 +731,6 @@ proc blocksDeliveryHandler*( codex_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64) - if peerCtx != nil: - if err =? catch(await self.payForBlocks(peerCtx, blocksDelivery)).errorOption: - warn "Error paying for blocks", err = err.msg - return - if err =? catch(await self.resolveBlocks(validatedBlocksDelivery)).errorOption: warn "Error resolving blocks", err = err.msg return @@ -790,7 +767,6 @@ proc wantListHandler*( except CatchableError as exc: # TODO: should not be necessary once we have proper exception tracking on the BlockStore interface false - price = @(self.pricing.get(Pricing(price: 0.u256)).price.toBytesBE) if e.cancel: # This is sort of expected if we sent the block to the peer, as we have removed @@ -806,7 +782,7 @@ proc wantListHandler*( trace "We HAVE the block", address = e.address presence.add( BlockPresence( - address: e.address, `type`: BlockPresenceType.Have, price: price + address: e.address, `type`: BlockPresenceType.Have ) ) else: @@ -814,7 +790,7 @@ proc wantListHandler*( if e.sendDontHave: presence.add( BlockPresence( - address: e.address, `type`: BlockPresenceType.DontHave, price: price + address: e.address, `type`: BlockPresenceType.DontHave ) ) @@ -856,30 +832,6 @@ proc wantListHandler*( except CancelledError as exc: #TODO: replace with CancelledError warn "Error processing want list", error = exc.msg -proc accountHandler*( - self: BlockExcEngine, peer: PeerId, account: Account -) {.async: (raises: []).} = - let context = self.peers.get(peer) - if context.isNil: - return - - context.account = account.some - -proc paymentHandler*( - self: BlockExcEngine, peer: PeerId, payment: SignedState -) {.async: (raises: []).} = - trace "Handling payments", peer - - without context =? self.peers.get(peer).option and account =? context.account: - trace "No context or account for peer", peer - return - - if channel =? context.paymentChannel: - let sender = account.address - discard self.wallet.acceptPayment(channel, Asset, sender, payment) - else: - context.paymentChannel = self.wallet.acceptChannel(payment).option - proc peerAddedHandler*( self: BlockExcEngine, peer: PeerId ) {.async: (raises: [CancelledError]).} = @@ -896,10 +848,6 @@ proc peerAddedHandler*( trace "Added peer", peers = self.peers.len await self.refreshBlockKnowledge(peerCtx) - if address =? self.pricing .? address: - trace "Sending account to peer", peer - await self.network.request.sendAccount(peer, Account(address: address)) - proc localLookup( self: BlockExcEngine, address: BlockAddress ): Future[?!BlockDelivery] {.async: (raises: [CancelledError]).} = @@ -1023,7 +971,6 @@ proc selectRandom*( proc new*( T: type BlockExcEngine, localStore: BlockStore, - wallet: WalletRef, network: BlockExcNetwork, discovery: DiscoveryEngine, advertiser: Advertiser, @@ -1041,7 +988,6 @@ proc new*( peers: peerStore, pendingBlocks: pendingBlocks, network: network, - wallet: wallet, concurrentTasks: concurrentTasks, trackedFutures: TrackedFutures(), maxBlocksPerMessage: maxBlocksPerMessage, @@ -1066,16 +1012,6 @@ proc new*( ): Future[void] {.async: (raises: []).} = self.blocksDeliveryHandler(peer, blocksDelivery) - proc accountHandler( - peer: PeerId, account: Account - ): Future[void] {.async: (raises: []).} = - self.accountHandler(peer, account) - - proc paymentHandler( - peer: PeerId, payment: SignedState - ): Future[void] {.async: (raises: []).} = - self.paymentHandler(peer, payment) - proc peerAddedHandler( peer: PeerId ): Future[void] {.async: (raises: [CancelledError]).} = @@ -1090,8 +1026,6 @@ proc new*( onWantList: blockWantListHandler, onBlocksDelivery: blocksDeliveryHandler, onPresence: blockPresenceHandler, - onAccount: accountHandler, - onPayment: paymentHandler, onPeerJoined: peerAddedHandler, onPeerDeparted: peerDepartedHandler, ) diff --git a/codex/blockexchange/engine/payments.nim b/codex/blockexchange/engine/payments.nim deleted file mode 100644 index 45259762..00000000 --- a/codex/blockexchange/engine/payments.nim +++ /dev/null @@ -1,46 +0,0 @@ -## Logos Storage -## Copyright (c) 2021 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [].} - -import std/math -import pkg/nitro -import pkg/questionable/results -import ../peers - -export nitro -export results - -const ChainId* = 0.u256 # invalid chain id for now -const Asset* = EthAddress.zero # invalid ERC20 asset address for now -const AmountPerChannel = (10'u64 ^ 18).u256 # 1 asset, ERC20 default is 18 decimals - -func openLedgerChannel*( - wallet: WalletRef, hub: EthAddress, asset: EthAddress -): ?!ChannelId = - wallet.openLedgerChannel(hub, ChainId, asset, AmountPerChannel) - -func getOrOpenChannel(wallet: WalletRef, peer: BlockExcPeerCtx): ?!ChannelId = - if channel =? peer.paymentChannel: - success channel - elif account =? peer.account: - let channel = ?wallet.openLedgerChannel(account.address, Asset) - peer.paymentChannel = channel.some - success channel - else: - failure "no account set for peer" - -func pay*(wallet: WalletRef, peer: BlockExcPeerCtx, amount: UInt256): ?!SignedState = - if account =? peer.account: - let asset = Asset - let receiver = account.address - let channel = ?wallet.getOrOpenChannel(peer) - wallet.pay(channel, asset, receiver, amount) - else: - failure "no account set for peer" diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index 4c6ca41a..91617d0f 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -20,12 +20,11 @@ import pkg/questionable/results import ../../blocktype as bt import ../../logutils import ../protobuf/blockexc as pb -import ../protobuf/payments import ../../utils/trackedfutures import ./networkpeer -export networkpeer, payments +export networkpeer logScope: topics = "codex blockexcnetwork" @@ -40,16 +39,12 @@ type proc(peer: PeerId, blocks: seq[BlockDelivery]) {.async: (raises: []).} BlockPresenceHandler* = proc(peer: PeerId, precense: seq[BlockPresence]) {.async: (raises: []).} - AccountHandler* = proc(peer: PeerId, account: Account) {.async: (raises: []).} - PaymentHandler* = proc(peer: PeerId, payment: SignedState) {.async: (raises: []).} PeerEventHandler* = proc(peer: PeerId) {.async: (raises: [CancelledError]).} BlockExcHandlers* = object onWantList*: WantListHandler onBlocksDelivery*: BlocksDeliveryHandler onPresence*: BlockPresenceHandler - onAccount*: AccountHandler - onPayment*: PaymentHandler onPeerJoined*: PeerEventHandler onPeerDeparted*: PeerEventHandler onPeerDropped*: PeerEventHandler @@ -72,18 +67,12 @@ type PresenceSender* = proc(peer: PeerId, presence: seq[BlockPresence]) {. async: (raises: [CancelledError]) .} - AccountSender* = - proc(peer: PeerId, account: Account) {.async: (raises: [CancelledError]).} - PaymentSender* = - proc(peer: PeerId, payment: SignedState) {.async: (raises: [CancelledError]).} BlockExcRequest* = object sendWantList*: WantListSender sendWantCancellations*: WantCancellationSender sendBlocksDelivery*: BlocksDeliverySender sendPresence*: PresenceSender - sendAccount*: AccountSender - sendPayment*: PaymentSender BlockExcNetwork* = ref object of LPProtocol peers*: Table[PeerId, NetworkPeer] @@ -207,40 +196,6 @@ proc sendBlockPresence*( b.send(id, Message(blockPresences: @presence)) -proc handleAccount( - network: BlockExcNetwork, peer: NetworkPeer, account: Account -) {.async: (raises: []).} = - ## Handle account info - ## - - if not network.handlers.onAccount.isNil: - await network.handlers.onAccount(peer.id, account) - -proc sendAccount*( - b: BlockExcNetwork, id: PeerId, account: Account -) {.async: (raw: true, raises: [CancelledError]).} = - ## Send account info to remote - ## - - b.send(id, Message(account: AccountMessage.init(account))) - -proc sendPayment*( - b: BlockExcNetwork, id: PeerId, payment: SignedState -) {.async: (raw: true, raises: [CancelledError]).} = - ## Send payment to remote - ## - - b.send(id, Message(payment: StateChannelUpdate.init(payment))) - -proc handlePayment( - network: BlockExcNetwork, peer: NetworkPeer, payment: SignedState -) {.async: (raises: []).} = - ## Handle payment - ## - - if not network.handlers.onPayment.isNil: - await network.handlers.onPayment(peer.id, payment) - proc rpcHandler( self: BlockExcNetwork, peer: NetworkPeer, msg: Message ) {.async: (raises: []).} = @@ -255,12 +210,6 @@ proc rpcHandler( if msg.blockPresences.len > 0: self.trackedFutures.track(self.handleBlockPresence(peer, msg.blockPresences)) - if account =? Account.init(msg.account): - self.trackedFutures.track(self.handleAccount(peer, account)) - - if payment =? SignedState.init(msg.payment): - self.trackedFutures.track(self.handlePayment(peer, payment)) - proc getOrCreatePeer(self: BlockExcNetwork, peer: PeerId): NetworkPeer = ## Creates or retrieves a BlockExcNetwork Peer ## @@ -413,23 +362,11 @@ proc new*( ): Future[void] {.async: (raw: true, raises: [CancelledError]).} = self.sendBlockPresence(id, presence) - proc sendAccount( - id: PeerId, account: Account - ): Future[void] {.async: (raw: true, raises: [CancelledError]).} = - self.sendAccount(id, account) - - proc sendPayment( - id: PeerId, payment: SignedState - ): Future[void] {.async: (raw: true, raises: [CancelledError]).} = - self.sendPayment(id, payment) - self.request = BlockExcRequest( sendWantList: sendWantList, sendWantCancellations: sendWantCancellations, sendBlocksDelivery: sendBlocksDelivery, sendPresence: sendPresence, - sendAccount: sendAccount, - sendPayment: sendPayment, ) self.init() diff --git a/codex/blockexchange/peers/peercontext.nim b/codex/blockexchange/peers/peercontext.nim index a02e8a89..9e29386f 100644 --- a/codex/blockexchange/peers/peercontext.nim +++ b/codex/blockexchange/peers/peercontext.nim @@ -13,18 +13,14 @@ import std/sets import pkg/libp2p import pkg/chronos -import pkg/nitro import pkg/questionable import ../protobuf/blockexc -import ../protobuf/payments import ../protobuf/presence import ../../blocktype import ../../logutils -export payments, nitro - const MinRefreshInterval = 1.seconds MaxRefreshBackoff = 36 # 36 seconds @@ -32,14 +28,12 @@ const type BlockExcPeerCtx* = ref object of RootObj id*: PeerId - blocks*: Table[BlockAddress, Presence] # remote peer have list including price + blocks*: Table[BlockAddress, Presence] # remote peer have list wantedBlocks*: HashSet[BlockAddress] # blocks that the peer wants exchanged*: int # times peer has exchanged with us refreshInProgress*: bool # indicates if a refresh is in progress lastRefresh*: Moment # last time we refreshed our knowledge of the blocks this peer has refreshBackoff*: int = 1 # backoff factor for refresh requests - account*: ?Account # ethereum account of this peer - paymentChannel*: ?ChannelId # payment channel id blocksSent*: HashSet[BlockAddress] # blocks sent to peer blocksRequested*: HashSet[BlockAddress] # pending block requests to this peer lastExchange*: Moment # last time peer has sent us a block @@ -105,14 +99,6 @@ func cleanPresence*(self: BlockExcPeerCtx, addresses: seq[BlockAddress]) = func cleanPresence*(self: BlockExcPeerCtx, address: BlockAddress) = self.cleanPresence(@[address]) -func price*(self: BlockExcPeerCtx, addresses: seq[BlockAddress]): UInt256 = - var price = 0.u256 - for a in addresses: - self.blocks.withValue(a, precense): - price += precense[].price - - price - proc blockRequestScheduled*(self: BlockExcPeerCtx, address: BlockAddress) = ## Adds a block the set of blocks that have been requested to this peer ## (its request schedule). diff --git a/codex/blockexchange/protobuf/blockexc.nim b/codex/blockexchange/protobuf/blockexc.nim index 9b1144ba..9cfea20d 100644 --- a/codex/blockexchange/protobuf/blockexc.nim +++ b/codex/blockexchange/protobuf/blockexc.nim @@ -17,7 +17,6 @@ import ../../blocktype export Message, protobufEncode, protobufDecode export Wantlist, WantType, WantListEntry export BlockDelivery, BlockPresenceType, BlockPresence -export AccountMessage, StateChannelUpdate proc hash*(e: WantListEntry): Hash = hash(e.address) diff --git a/codex/blockexchange/protobuf/message.nim b/codex/blockexchange/protobuf/message.nim index 03c9dd00..507831f7 100644 --- a/codex/blockexchange/protobuf/message.nim +++ b/codex/blockexchange/protobuf/message.nim @@ -51,10 +51,6 @@ type BlockPresence* = object address*: BlockAddress `type`*: BlockPresenceType - price*: seq[byte] # Amount of assets to pay for the block (UInt256) - - AccountMessage* = object - address*: seq[byte] # Ethereum address to which payments should be made StateChannelUpdate* = object update*: seq[byte] # Signed Nitro state, serialized as JSON @@ -64,8 +60,6 @@ type payload*: seq[BlockDelivery] blockPresences*: seq[BlockPresence] pendingBytes*: uint - account*: AccountMessage - payment*: StateChannelUpdate # # Encoding Message into seq[byte] in Protobuf format @@ -115,19 +109,6 @@ proc write*(pb: var ProtoBuffer, field: int, value: BlockPresence) = var ipb = initProtoBuffer() ipb.write(1, value.address) ipb.write(2, value.`type`.uint) - ipb.write(3, value.price) - ipb.finish() - pb.write(field, ipb) - -proc write*(pb: var ProtoBuffer, field: int, value: AccountMessage) = - var ipb = initProtoBuffer() - ipb.write(1, value.address) - ipb.finish() - pb.write(field, ipb) - -proc write*(pb: var ProtoBuffer, field: int, value: StateChannelUpdate) = - var ipb = initProtoBuffer() - ipb.write(1, value.update) ipb.finish() pb.write(field, ipb) @@ -135,12 +116,10 @@ proc protobufEncode*(value: Message): seq[byte] = var ipb = initProtoBuffer() ipb.write(1, value.wantList) for v in value.payload: - ipb.write(3, v) + ipb.write(3, v) # is this meant to be 2? for v in value.blockPresences: ipb.write(4, v) ipb.write(5, value.pendingBytes) - ipb.write(6, value.account) - ipb.write(7, value.payment) ipb.finish() ipb.buffer @@ -240,19 +219,6 @@ proc decode*(_: type BlockPresence, pb: ProtoBuffer): ProtoResult[BlockPresence] value.address = ?BlockAddress.decode(ipb) if ?pb.getField(2, field): value.`type` = BlockPresenceType(field) - discard ?pb.getField(3, value.price) - ok(value) - -proc decode*(_: type AccountMessage, pb: ProtoBuffer): ProtoResult[AccountMessage] = - var value = AccountMessage() - discard ?pb.getField(1, value.address) - ok(value) - -proc decode*( - _: type StateChannelUpdate, pb: ProtoBuffer -): ProtoResult[StateChannelUpdate] = - var value = StateChannelUpdate() - discard ?pb.getField(1, value.update) ok(value) proc protobufDecode*(_: type Message, msg: seq[byte]): ProtoResult[Message] = @@ -263,15 +229,11 @@ proc protobufDecode*(_: type Message, msg: seq[byte]): ProtoResult[Message] = sublist: seq[seq[byte]] if ?pb.getField(1, ipb): value.wantList = ?WantList.decode(ipb) - if ?pb.getRepeatedField(3, sublist): + if ?pb.getRepeatedField(3, sublist): # meant to be 2? for item in sublist: value.payload.add(?BlockDelivery.decode(initProtoBuffer(item))) if ?pb.getRepeatedField(4, sublist): for item in sublist: value.blockPresences.add(?BlockPresence.decode(initProtoBuffer(item))) discard ?pb.getField(5, value.pendingBytes) - if ?pb.getField(6, ipb): - value.account = ?AccountMessage.decode(ipb) - if ?pb.getField(7, ipb): - value.payment = ?StateChannelUpdate.decode(ipb) ok(value) diff --git a/codex/blockexchange/protobuf/message.proto b/codex/blockexchange/protobuf/message.proto index ee24c797..bdd63276 100644 --- a/codex/blockexchange/protobuf/message.proto +++ b/codex/blockexchange/protobuf/message.proto @@ -38,21 +38,10 @@ message Message { message BlockPresence { bytes cid = 1; BlockPresenceType type = 2; - bytes price = 3; // Amount of assets to pay for the block (UInt256) - } - - message AccountMessage { - bytes address = 1; // Ethereum address to which payments should be made - } - - message StateChannelUpdate { - bytes update = 1; // Signed Nitro state, serialized as JSON } Wantlist wantlist = 1; - repeated Block payload = 3; + repeated Block payload = 3; // what happened to 2? repeated BlockPresence blockPresences = 4; int32 pendingBytes = 5; - AccountMessage account = 6; - StateChannelUpdate payment = 7; } diff --git a/codex/blockexchange/protobuf/payments.nim b/codex/blockexchange/protobuf/payments.nim deleted file mode 100644 index 885562c4..00000000 --- a/codex/blockexchange/protobuf/payments.nim +++ /dev/null @@ -1,38 +0,0 @@ -{.push raises: [].} - -import pkg/stew/byteutils -import pkg/stint -import pkg/nitro -import pkg/questionable -import ./blockexc - -export AccountMessage -export StateChannelUpdate - -export stint -export nitro - -type Account* = object - address*: EthAddress - -func init*(_: type AccountMessage, account: Account): AccountMessage = - AccountMessage(address: @(account.address.toArray)) - -func parse(_: type EthAddress, bytes: seq[byte]): ?EthAddress = - var address: array[20, byte] - if bytes.len != address.len: - return EthAddress.none - for i in 0 ..< address.len: - address[i] = bytes[i] - EthAddress(address).some - -func init*(_: type Account, message: AccountMessage): ?Account = - without address =? EthAddress.parse(message.address): - return none Account - some Account(address: address) - -func init*(_: type StateChannelUpdate, state: SignedState): StateChannelUpdate = - StateChannelUpdate(update: state.toJson.toBytes) - -proc init*(_: type SignedState, update: StateChannelUpdate): ?SignedState = - SignedState.fromJson(string.fromBytes(update.update)) diff --git a/codex/blockexchange/protobuf/presence.nim b/codex/blockexchange/protobuf/presence.nim index 3b24a570..238f38b2 100644 --- a/codex/blockexchange/protobuf/presence.nim +++ b/codex/blockexchange/protobuf/presence.nim @@ -17,7 +17,6 @@ type Presence* = object address*: BlockAddress have*: bool - price*: UInt256 func parse(_: type UInt256, bytes: seq[byte]): ?UInt256 = if bytes.len > 32: @@ -25,18 +24,14 @@ func parse(_: type UInt256, bytes: seq[byte]): ?UInt256 = UInt256.fromBytesBE(bytes).some func init*(_: type Presence, message: PresenceMessage): ?Presence = - without price =? UInt256.parse(message.price): - return none Presence some Presence( address: message.address, have: message.`type` == BlockPresenceType.Have, - price: price, ) func init*(_: type PresenceMessage, presence: Presence): PresenceMessage = PresenceMessage( address: presence.address, `type`: if presence.have: BlockPresenceType.Have else: BlockPresenceType.DontHave, - price: @(presence.price.toBytesBE), ) diff --git a/codex/codex.nim b/codex/codex.nim index 73575c53..b95bf26a 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -20,10 +20,8 @@ import pkg/presto import pkg/libp2p import pkg/confutils import pkg/confutils/defs -import pkg/nitro import pkg/stew/io2 import pkg/datastore -import pkg/ethers except Rng import pkg/stew/io2 import ./node @@ -31,15 +29,10 @@ import ./conf import ./rng as random import ./rest/api import ./stores -import ./slots import ./blockexchange import ./utils/fileutils -import ./erasure import ./discovery -import ./contracts import ./systemclock -import ./contracts/clock -import ./contracts/deployment import ./utils/addrutils import ./namespaces import ./codextypes @@ -60,7 +53,6 @@ type isStarted: bool CodexPrivateKey* = libp2p.PrivateKey # alias - EthWallet = ethers.Wallet func config*(self: CodexServer): CodexConf = return self.config @@ -71,103 +63,6 @@ func node*(self: CodexServer): CodexNodeRef = func repoStore*(self: CodexServer): RepoStore = return self.repoStore -proc waitForSync(provider: Provider): Future[void] {.async.} = - var sleepTime = 1 - trace "Checking sync state of Ethereum provider..." - while await provider.isSyncing: - notice "Waiting for Ethereum provider to sync..." - await sleepAsync(sleepTime.seconds) - if sleepTime < 10: - inc sleepTime - trace "Ethereum provider is synced." - -proc bootstrapInteractions(s: CodexServer): Future[void] {.async.} = - ## bootstrap interactions and return contracts - ## using clients, hosts, validators pairings - ## - let - config = s.config - repo = s.repoStore - - if config.persistence: - if not config.ethAccount.isSome and not config.ethPrivateKey.isSome: - error "Persistence enabled, but no Ethereum account was set" - quit QuitFailure - - let provider = JsonRpcProvider.new( - config.ethProvider, maxPriorityFeePerGas = config.maxPriorityFeePerGas.u256 - ) - await waitForSync(provider) - var signer: Signer - if account =? config.ethAccount: - signer = provider.getSigner(account) - elif keyFile =? config.ethPrivateKey: - without isSecure =? checkSecureFile(keyFile): - error "Could not check file permissions: does Ethereum private key file exist?" - quit QuitFailure - if not isSecure: - error "Ethereum private key file does not have safe file permissions" - quit QuitFailure - without key =? keyFile.readAllChars(): - error "Unable to read Ethereum private key file" - quit QuitFailure - without wallet =? EthWallet.new(key.strip(), provider): - error "Invalid Ethereum private key in file" - quit QuitFailure - signer = wallet - - let deploy = Deployment.new(provider, config.marketplaceAddress) - without marketplaceAddress =? await deploy.address(Marketplace): - error "No Marketplace address was specified or there is no known address for the current network" - quit QuitFailure - - let marketplace = Marketplace.new(marketplaceAddress, signer) - let market = OnChainMarket.new( - marketplace, config.rewardRecipient, config.marketplaceRequestCacheSize - ) - let clock = OnChainClock.new(provider) - - var client: ?ClientInteractions - var host: ?HostInteractions - var validator: ?ValidatorInteractions - - if config.validator or config.persistence: - s.codexNode.clock = clock - else: - s.codexNode.clock = SystemClock() - - # This is used for simulation purposes. Normal nodes won't be compiled with this flag - # and hence the proof failure will always be 0. - when storage_enable_proof_failures: - let proofFailures = config.simulateProofFailures - if proofFailures > 0: - warn "Enabling proof failure simulation!" - else: - let proofFailures = 0 - if config.simulateProofFailures > 0: - warn "Proof failure simulation is not enabled for this build! Configuration ignored" - - if error =? (await market.loadConfig()).errorOption: - fatal "Cannot load market configuration", error = error.msg - quit QuitFailure - - let purchasing = Purchasing.new(market, clock) - let sales = Sales.new(market, clock, repo, proofFailures) - client = some ClientInteractions.new(clock, purchasing) - host = some HostInteractions.new(clock, sales) - - if config.validator: - without validationConfig =? - ValidationConfig.init( - config.validatorMaxSlots, config.validatorGroups, config.validatorGroupIndex - ), err: - error "Invalid validation parameters", err = err.msg - quit QuitFailure - let validation = Validation.new(clock, market, validationConfig) - validator = some ValidatorInteractions.new(clock, validation) - - s.codexNode.contracts = (client, host, validator) - proc start*(s: CodexServer) {.async.} = if s.isStarted: warn "Storage server already started, skipping" @@ -187,7 +82,6 @@ proc start*(s: CodexServer) {.async.} = s.codexNode.discovery.updateAnnounceRecord(announceAddrs) s.codexNode.discovery.updateDhtRecord(discoveryAddrs) - await s.bootstrapInteractions() await s.codexNode.start() if s.restServer != nil: @@ -297,7 +191,6 @@ proc new*( store = discoveryStore, ) - wallet = WalletRef.new(EthPrivateKey.random()) network = BlockExcNetwork.new(switch) repoData = @@ -342,23 +235,15 @@ proc new*( blockDiscovery = DiscoveryEngine.new(repoStore, peerStore, network, discovery, pendingBlocks) engine = BlockExcEngine.new( - repoStore, wallet, network, blockDiscovery, advertiser, peerStore, pendingBlocks + repoStore, network, blockDiscovery, advertiser, peerStore, pendingBlocks ) store = NetworkStore.new(engine, repoStore) - prover = - if config.prover: - let backend = - config.initializeBackend().expect("Unable to create prover backend.") - some Prover.new(store, backend, config.numProofSamples) - else: - none Prover codexNode = CodexNodeRef.new( switch = switch, networkStore = store, engine = engine, discovery = discovery, - prover = prover, taskPool = taskpool, ) diff --git a/codex/conf.nim b/codex/conf.nim index bc8b5506..f92ce990 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -31,7 +31,6 @@ import pkg/metrics import pkg/metrics/chronos_httpserver import pkg/stew/byteutils import pkg/libp2p -import pkg/ethers import pkg/questionable import pkg/questionable/results import pkg/stew/base64 @@ -45,16 +44,13 @@ import ./utils import ./nat import ./utils/natutils -from ./contracts/config import DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas -from ./validationconfig import MaxSlots, ValidationGroups from ./blockexchange/engine/pendingblocks import DefaultBlockRetries export units, net, codextypes, logutils, completeCmdArg, parseCmdArg, NatConfig -export ValidationGroups, MaxSlots export DefaultQuotaBytes, DefaultBlockTtl, DefaultBlockInterval, DefaultNumBlocksPerInterval, - DefaultRequestCacheSize, DefaultMaxPriorityFeePerGas, DefaultBlockRetries + DefaultBlockRetries type ThreadCount* = distinct Natural @@ -73,7 +69,6 @@ proc defaultDataDir*(): string = const storage_enable_api_debug_peers* {.booldefine.} = false - storage_enable_proof_failures* {.booldefine.} = false storage_enable_log_counter* {.booldefine.} = false DefaultThreadCount* = ThreadCount(0) @@ -83,10 +78,6 @@ type noCmd persistence - PersistenceCmd* {.pure.} = enum - noCmd - prover - LogKind* {.pure.} = enum Auto = "auto" Colors = "colors" @@ -286,204 +277,12 @@ type desc: "Logs to file", defaultValue: string.none, name: "log-file", hidden .}: Option[string] - case cmd* {.defaultValue: noCmd, command.}: StartUpCmd - of persistence: - ethProvider* {. - desc: "The URL of the JSON-RPC API of the Ethereum node", - defaultValue: "ws://localhost:8545", - name: "eth-provider" - .}: string - - ethAccount* {. - desc: "The Ethereum account that is used for storage contracts", - defaultValue: EthAddress.none, - defaultValueDesc: "", - name: "eth-account" - .}: Option[EthAddress] - - ethPrivateKey* {. - desc: "File containing Ethereum private key for storage contracts", - defaultValue: string.none, - defaultValueDesc: "", - name: "eth-private-key" - .}: Option[string] - - marketplaceAddress* {. - desc: "Address of deployed Marketplace contract", - defaultValue: EthAddress.none, - defaultValueDesc: "", - name: "marketplace-address" - .}: Option[EthAddress] - - # TODO: should go behind a feature flag - simulateProofFailures* {. - desc: "Simulates proof failures once every N proofs. 0 = disabled.", - defaultValue: 0, - name: "simulate-proof-failures", - hidden - .}: int - - validator* {. - desc: "Enables validator, requires an Ethereum node", - defaultValue: false, - name: "validator" - .}: bool - - validatorMaxSlots* {. - desc: "Maximum number of slots that the validator monitors", - longDesc: - "If set to 0, the validator will not limit " & - "the maximum number of slots it monitors", - defaultValue: 1000, - name: "validator-max-slots" - .}: MaxSlots - - validatorGroups* {. - desc: "Slot validation groups", - longDesc: - "A number indicating total number of groups into " & - "which the whole slot id space will be divided. " & - "The value must be in the range [2, 65535]. " & - "If not provided, the validator will observe " & - "the whole slot id space and the value of " & - "the --validator-group-index parameter will be ignored. " & - "Powers of twos are advised for even distribution", - defaultValue: ValidationGroups.none, - name: "validator-groups" - .}: Option[ValidationGroups] - - validatorGroupIndex* {. - desc: "Slot validation group index", - longDesc: - "The value provided must be in the range " & - "[0, validatorGroups). Ignored when --validator-groups " & - "is not provided. Only slot ids satisfying condition " & - "[(slotId mod validationGroups) == groupIndex] will be " & - "observed by the validator", - defaultValue: 0, - name: "validator-group-index" - .}: uint16 - - rewardRecipient* {. - desc: "Address to send payouts to (eg rewards and refunds)", - name: "reward-recipient" - .}: Option[EthAddress] - - marketplaceRequestCacheSize* {. - desc: - "Maximum number of StorageRequests kept in memory." & - "Reduces fetching of StorageRequest data from the contract.", - defaultValue: DefaultRequestCacheSize, - defaultValueDesc: $DefaultRequestCacheSize, - name: "request-cache-size", - hidden - .}: uint16 - - maxPriorityFeePerGas* {. - desc: - "Sets the default maximum priority fee per gas for Ethereum EIP-1559 transactions, in wei, when not provided by the network.", - defaultValue: DefaultMaxPriorityFeePerGas, - defaultValueDesc: $DefaultMaxPriorityFeePerGas, - name: "max-priority-fee-per-gas", - hidden - .}: uint64 - - case persistenceCmd* {.defaultValue: noCmd, command.}: PersistenceCmd - of PersistenceCmd.prover: - circuitDir* {. - desc: "Directory where Storage will store proof circuit data", - defaultValue: defaultDataDir() / "circuits", - defaultValueDesc: "data/circuits", - abbr: "cd", - name: "circuit-dir" - .}: OutDir - - circomR1cs* {. - desc: "The r1cs file for the storage circuit", - defaultValue: defaultDataDir() / "circuits" / "proof_main.r1cs", - defaultValueDesc: "data/circuits/proof_main.r1cs", - name: "circom-r1cs" - .}: InputFile - - circomWasm* {. - desc: "The wasm file for the storage circuit", - defaultValue: defaultDataDir() / "circuits" / "proof_main.wasm", - defaultValueDesc: "data/circuits/proof_main.wasm", - name: "circom-wasm" - .}: InputFile - - circomZkey* {. - desc: "The zkey file for the storage circuit", - defaultValue: defaultDataDir() / "circuits" / "proof_main.zkey", - defaultValueDesc: "data/circuits/proof_main.zkey", - name: "circom-zkey" - .}: InputFile - - # TODO: should probably be hidden and behind a feature flag - circomNoZkey* {. - desc: "Ignore the zkey file - use only for testing!", - defaultValue: false, - name: "circom-no-zkey" - .}: bool - - numProofSamples* {. - desc: "Number of samples to prove", - defaultValue: DefaultSamplesNum, - defaultValueDesc: $DefaultSamplesNum, - name: "proof-samples" - .}: int - - maxSlotDepth* {. - desc: "The maximum depth of the slot tree", - defaultValue: DefaultMaxSlotDepth, - defaultValueDesc: $DefaultMaxSlotDepth, - name: "max-slot-depth" - .}: int - - maxDatasetDepth* {. - desc: "The maximum depth of the dataset tree", - defaultValue: DefaultMaxDatasetDepth, - defaultValueDesc: $DefaultMaxDatasetDepth, - name: "max-dataset-depth" - .}: int - - maxBlockDepth* {. - desc: "The maximum depth of the network block merkle tree", - defaultValue: DefaultBlockDepth, - defaultValueDesc: $DefaultBlockDepth, - name: "max-block-depth" - .}: int - - maxCellElms* {. - desc: "The maximum number of elements in a cell", - defaultValue: DefaultCellElms, - defaultValueDesc: $DefaultCellElms, - name: "max-cell-elements" - .}: int - of PersistenceCmd.noCmd: - discard - of StartUpCmd.noCmd: - discard # end of persistence - - EthAddress* = ethers.Address - -logutils.formatIt(LogFormat.textLines, EthAddress): - it.short0xHexLog -logutils.formatIt(LogFormat.json, EthAddress): - %it - func defaultAddress*(conf: CodexConf): IpAddress = result = static parseIpAddress("127.0.0.1") func defaultNatConfig*(): NatConfig = result = NatConfig(hasExtIp: false, nat: NatStrategy.NatAny) -func persistence*(self: CodexConf): bool = - self.cmd == StartUpCmd.persistence - -func prover*(self: CodexConf): bool = - self.persistence and self.persistenceCmd == PersistenceCmd.prover - proc getCodexVersion(): string = let tag = strip(staticExec("git describe --tags --abbrev=0")) if tag.isEmptyOrWhitespace: @@ -495,23 +294,16 @@ proc getCodexRevision(): string = var res = strip(staticExec("git rev-parse --short HEAD")) return res -proc getCodexContractsRevision(): string = - let res = - strip(staticExec("git rev-parse --short HEAD:vendor/logos-storage-contracts-eth")) - return res - proc getNimBanner(): string = staticExec("nim --version | grep Version") const codexVersion* = getCodexVersion() codexRevision* = getCodexRevision() - codexContractsRevision* = getCodexContractsRevision() nimBanner* = getNimBanner() codexFullVersion* = - "Storage version: " & codexVersion & "\p" & "Storage revision: " & codexRevision & - "\p" & "Storage contracts revision: " & codexContractsRevision & "\p" & nimBanner + "Storage version: " & codexVersion & "\p" & "Storage revision: " & codexRevision & "\p" proc parseCmdArg*( T: typedesc[MultiAddress], input: string @@ -593,9 +385,6 @@ proc parseCmdArg*(T: type NatConfig, p: string): T = proc completeCmdArg*(T: type NatConfig, val: string): seq[string] = return @[] -proc parseCmdArg*(T: type EthAddress, address: string): T = - EthAddress.init($address).get() - func parse*(T: type NBytes, p: string): Result[NBytes, string] = var num = 0'i64 let count = parseSize(p, num, alwaysBin = true) @@ -618,11 +407,6 @@ proc parseCmdArg*(T: type Duration, val: string): T = quit QuitFailure dur -proc readValue*( - r: var TomlReader, val: var EthAddress -) {.raises: [SerializationError, IOError].} = - val = EthAddress.init(r.readValue(string)).get() - proc readValue*(r: var TomlReader, val: var SignedPeerRecord) = without uri =? r.readValue(string).catch, err: error "invalid SignedPeerRecord configuration value", error = err.msg @@ -687,9 +471,6 @@ proc readValue*( raise newException(SerializationError, err.msg) # no idea why confutils needs this: -proc completeCmdArg*(T: type EthAddress, val: string): seq[string] = - discard - proc completeCmdArg*(T: type NBytes, val: string): seq[string] = discard diff --git a/codex/contracts.nim b/codex/contracts.nim deleted file mode 100644 index 512571a8..00000000 --- a/codex/contracts.nim +++ /dev/null @@ -1,11 +0,0 @@ -import contracts/requests -import contracts/marketplace -import contracts/market -import contracts/interactions -import contracts/provider - -export requests -export marketplace -export market -export interactions -export provider diff --git a/codex/contracts/Readme.md b/codex/contracts/Readme.md deleted file mode 100644 index 2a746863..00000000 --- a/codex/contracts/Readme.md +++ /dev/null @@ -1,148 +0,0 @@ -Logos Storage Contracts in Nim -======================= - -Nim API for the [Logos Storage smart contracts][1]. - -Usage ------ - -For a global overview of the steps involved in starting and fulfilling a -storage contract, see [Logos Storage Contracts][1]. - -Smart contract --------------- - -Connecting to the smart contract on an Ethereum node: - -```nim -import codex/contracts -import ethers - -let address = # fill in address where the contract was deployed -let provider = JsonRpcProvider.new("ws://localhost:8545") -let marketplace = Marketplace.new(address, provider) -``` - -Setup client and host so that they can sign transactions; here we use the first -two accounts on the Ethereum node: - -```nim -let accounts = await provider.listAccounts() -let client = provider.getSigner(accounts[0]) -let host = provider.getSigner(accounts[1]) -``` - -Storage requests ----------------- - -Creating a request for storage: - -```nim -let request : StorageRequest = ( - client: # address of the client requesting storage - duration: # duration of the contract in seconds - size: # size in bytes - contentHash: # SHA256 hash of the content that's going to be stored - proofProbability: # require a storage proof roughly once every N periods - maxPrice: # maximum price the client is willing to pay - expiry: # expiration time of the request (in unix time) - nonce: # random nonce to differentiate between similar requests -) -``` - -When a client wants to submit this request to the network, it needs to pay the -maximum price to the smart contract in advance. The difference between the -maximum price and the offered price will be reimbursed later. - -Once the payment has been prepared, the client can submit the request to the -network: - -```nim -await storage - .connect(client) - .requestStorage(request) -``` - -Storage offers --------------- - -Creating a storage offer: - -```nim -let offer: StorageOffer = ( - host: # address of the host that is offering storage - requestId: request.id, - price: # offered price (in number of tokens) - expiry: # expiration time of the offer (in unix time) -) -``` - -Hosts submits an offer: - -```nim -await storage - .connect(host) - .offerStorage(offer) -``` - -Client selects an offer: - -```nim -await storage - .connect(client) - .selectOffer(offer.id) -``` - -Starting and finishing a storage contract ------------------------------------------ - -The host whose offer got selected can start the storage contract once it -received the data that needs to be stored: - -```nim -await storage - .connect(host) - .startContract(offer.id) -``` - -Once the storage contract is finished, the host can release payment: - -```nim -await storage - .connect(host) - .finishContract(id) -``` - -Storage proofs --------------- - -Time is divided into periods, and each period a storage proof may be required -from the host. The odds of requiring a storage proof are negotiated through the -storage request. For more details about the timing of storage proofs, please -refer to the [design document][2]. - -At the start of each period of time, the host can check whether a storage proof -is required: - -```nim -let isProofRequired = await storage.isProofRequired(offer.id) -``` - -If a proof is required, the host can submit it before the end of the period: - -```nim -await storage - .connect(host) - .submitProof(id, proof) -``` - -If a proof is not submitted, then a validator can mark a proof as missing: - -```nim -await storage - .connect(validator) - .markProofAsMissing(id, period) -``` - -[1]: https://github.com/logos-storage/logos-storage-contracts-eth/ -[2]: https://github.com/logos-storage/logos-storage-research/blob/master/design/storage-proof-timing.md diff --git a/codex/contracts/clock.nim b/codex/contracts/clock.nim deleted file mode 100644 index 1d4f57ba..00000000 --- a/codex/contracts/clock.nim +++ /dev/null @@ -1,82 +0,0 @@ -{.push raises: [].} - -import std/times -import pkg/ethers -import pkg/questionable -import pkg/chronos -import pkg/stint -import ../clock -import ../conf -import ../utils/trackedfutures - -export clock - -logScope: - topics = "contracts clock" - -type OnChainClock* = ref object of Clock - provider: Provider - subscription: Subscription - offset: times.Duration - blockNumber: UInt256 - started: bool - newBlock: AsyncEvent - trackedFutures: TrackedFutures - -proc new*(_: type OnChainClock, provider: Provider): OnChainClock = - OnChainClock( - provider: provider, newBlock: newAsyncEvent(), trackedFutures: TrackedFutures() - ) - -proc update(clock: OnChainClock, blck: Block) = - if number =? blck.number and number > clock.blockNumber: - let blockTime = initTime(blck.timestamp.truncate(int64), 0) - let computerTime = getTime() - clock.offset = blockTime - computerTime - clock.blockNumber = number - trace "updated clock", - blockTime = blck.timestamp, blockNumber = number, offset = clock.offset - clock.newBlock.fire() - -proc update(clock: OnChainClock) {.async: (raises: []).} = - try: - if latest =? (await clock.provider.getBlock(BlockTag.latest)): - clock.update(latest) - except CatchableError as error: - debug "error updating clock: ", error = error.msg - -method start*(clock: OnChainClock) {.async.} = - if clock.started: - return - - proc onBlock(blckResult: ?!Block) = - if eventError =? blckResult.errorOption: - error "There was an error in block subscription", msg = eventError.msg - return - - # ignore block parameter; hardhat may call this with pending blocks - clock.trackedFutures.track(clock.update()) - - await clock.update() - - clock.subscription = await clock.provider.subscribe(onBlock) - clock.started = true - -method stop*(clock: OnChainClock) {.async.} = - if not clock.started: - return - - await clock.subscription.unsubscribe() - await clock.trackedFutures.cancelTracked() - clock.started = false - -method now*(clock: OnChainClock): SecondsSince1970 = - doAssert clock.started, "clock should be started before calling now()" - return toUnix(getTime() + clock.offset) - -method waitUntil*( - clock: OnChainClock, time: SecondsSince1970 -) {.async: (raises: [CancelledError]).} = - while (let difference = time - clock.now(); difference > 0): - clock.newBlock.clear() - discard await clock.newBlock.wait().withTimeout(chronos.seconds(difference)) diff --git a/codex/contracts/config.nim b/codex/contracts/config.nim deleted file mode 100644 index 5ff9bfe0..00000000 --- a/codex/contracts/config.nim +++ /dev/null @@ -1,104 +0,0 @@ -import pkg/contractabi -import pkg/ethers/contracts/fields -import pkg/questionable/results - -export contractabi - -const DefaultRequestCacheSize* = 128.uint16 -const DefaultMaxPriorityFeePerGas* = 1_000_000_000.uint64 - -type - MarketplaceConfig* = object - collateral*: CollateralConfig - proofs*: ProofConfig - reservations*: SlotReservationsConfig - requestDurationLimit*: uint64 - - CollateralConfig* = object - repairRewardPercentage*: uint8 - # percentage of remaining collateral slot has after it has been freed - maxNumberOfSlashes*: uint8 # frees slot when the number of slashes reaches this value - slashPercentage*: uint8 # percentage of the collateral that is slashed - validatorRewardPercentage*: uint8 - # percentage of the slashed amount going to the validators - - ProofConfig* = object - period*: uint64 # proofs requirements are calculated per period (in seconds) - timeout*: uint64 # mark proofs as missing before the timeout (in seconds) - downtime*: uint8 # ignore this much recent blocks for proof requirements - downtimeProduct*: uint8 - zkeyHash*: string # hash of the zkey file which is linked to the verifier - # Ensures the pointer does not remain in downtime for many consecutive - # periods. For each period increase, move the pointer `pointerProduct` - # blocks. Should be a prime number to ensure there are no cycles. - - SlotReservationsConfig* = object - maxReservations*: uint8 - -func fromTuple(_: type ProofConfig, tupl: tuple): ProofConfig = - ProofConfig( - period: tupl[0], - timeout: tupl[1], - downtime: tupl[2], - downtimeProduct: tupl[3], - zkeyHash: tupl[4], - ) - -func fromTuple(_: type SlotReservationsConfig, tupl: tuple): SlotReservationsConfig = - SlotReservationsConfig(maxReservations: tupl[0]) - -func fromTuple(_: type CollateralConfig, tupl: tuple): CollateralConfig = - CollateralConfig( - repairRewardPercentage: tupl[0], - maxNumberOfSlashes: tupl[1], - slashPercentage: tupl[2], - validatorRewardPercentage: tupl[3], - ) - -func fromTuple(_: type MarketplaceConfig, tupl: tuple): MarketplaceConfig = - MarketplaceConfig( - collateral: tupl[0], - proofs: tupl[1], - reservations: tupl[2], - requestDurationLimit: tupl[3], - ) - -func solidityType*(_: type SlotReservationsConfig): string = - solidityType(SlotReservationsConfig.fieldTypes) - -func solidityType*(_: type ProofConfig): string = - solidityType(ProofConfig.fieldTypes) - -func solidityType*(_: type CollateralConfig): string = - solidityType(CollateralConfig.fieldTypes) - -func solidityType*(_: type MarketplaceConfig): string = - solidityType(MarketplaceConfig.fieldTypes) - -func encode*(encoder: var AbiEncoder, slot: SlotReservationsConfig) = - encoder.write(slot.fieldValues) - -func encode*(encoder: var AbiEncoder, slot: ProofConfig) = - encoder.write(slot.fieldValues) - -func encode*(encoder: var AbiEncoder, slot: CollateralConfig) = - encoder.write(slot.fieldValues) - -func encode*(encoder: var AbiEncoder, slot: MarketplaceConfig) = - encoder.write(slot.fieldValues) - -func decode*(decoder: var AbiDecoder, T: type ProofConfig): ?!T = - let tupl = ?decoder.read(ProofConfig.fieldTypes) - success ProofConfig.fromTuple(tupl) - -func decode*(decoder: var AbiDecoder, T: type SlotReservationsConfig): ?!T = - let tupl = ?decoder.read(SlotReservationsConfig.fieldTypes) - success SlotReservationsConfig.fromTuple(tupl) - -func decode*(decoder: var AbiDecoder, T: type CollateralConfig): ?!T = - let tupl = ?decoder.read(CollateralConfig.fieldTypes) - success CollateralConfig.fromTuple(tupl) - -func decode*(decoder: var AbiDecoder, T: type MarketplaceConfig): ?!T = - let tupl = ?decoder.read(MarketplaceConfig.fieldTypes) - success MarketplaceConfig.fromTuple(tupl) diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim deleted file mode 100644 index 55b4b55c..00000000 --- a/codex/contracts/deployment.nim +++ /dev/null @@ -1,51 +0,0 @@ -import std/os -import std/tables -import pkg/ethers -import pkg/questionable - -import ../conf -import ../logutils -import ./marketplace - -type Deployment* = ref object - provider: Provider - marketplaceAddressOverride: ?Address - -const knownAddresses = { - # Hardhat localhost network - "31337": - {"Marketplace": Address.init("0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44")}.toTable, - # Taiko Alpha-3 Testnet - "167005": - {"Marketplace": Address.init("0x948CF9291b77Bd7ad84781b9047129Addf1b894F")}.toTable, - # Codex Testnet - Jun 19 2025 13:11:56 PM (+00:00 UTC) - "789987": - {"Marketplace": Address.init("0x5378a4EA5dA2a548ce22630A3AE74b052000C62D")}.toTable, - # Linea (Status) - "1660990954": - {"Marketplace": Address.init("0x34F606C65869277f236ce07aBe9af0B8c88F486B")}.toTable, -}.toTable - -proc getKnownAddress(T: type, chainId: UInt256): ?Address = - let id = chainId.toString(10) - notice "Looking for well-known contract address with ChainID ", chainId = id - - if not (id in knownAddresses): - return none Address - - return knownAddresses[id].getOrDefault($T, Address.none) - -proc new*( - _: type Deployment, - provider: Provider, - marketplaceAddressOverride: ?Address = none Address, -): Deployment = - Deployment(provider: provider, marketplaceAddressOverride: marketplaceAddressOverride) - -proc address*(deployment: Deployment, contract: type): Future[?Address] {.async.} = - when contract is Marketplace: - if address =? deployment.marketplaceAddressOverride: - return some address - - let chainId = await deployment.provider.getChainId() - return contract.getKnownAddress(chainId) diff --git a/codex/contracts/interactions.nim b/codex/contracts/interactions.nim deleted file mode 100644 index 13eae8a0..00000000 --- a/codex/contracts/interactions.nim +++ /dev/null @@ -1,9 +0,0 @@ -import ./interactions/interactions -import ./interactions/hostinteractions -import ./interactions/clientinteractions -import ./interactions/validatorinteractions - -export interactions -export hostinteractions -export clientinteractions -export validatorinteractions diff --git a/codex/contracts/interactions/clientinteractions.nim b/codex/contracts/interactions/clientinteractions.nim deleted file mode 100644 index df81da11..00000000 --- a/codex/contracts/interactions/clientinteractions.nim +++ /dev/null @@ -1,26 +0,0 @@ -import pkg/ethers - -import ../../purchasing -import ../../logutils -import ../market -import ../clock -import ./interactions - -export purchasing -export logutils - -type ClientInteractions* = ref object of ContractInteractions - purchasing*: Purchasing - -proc new*( - _: type ClientInteractions, clock: OnChainClock, purchasing: Purchasing -): ClientInteractions = - ClientInteractions(clock: clock, purchasing: purchasing) - -proc start*(self: ClientInteractions) {.async.} = - await procCall ContractInteractions(self).start() - await self.purchasing.start() - -proc stop*(self: ClientInteractions) {.async.} = - await self.purchasing.stop() - await procCall ContractInteractions(self).stop() diff --git a/codex/contracts/interactions/hostinteractions.nim b/codex/contracts/interactions/hostinteractions.nim deleted file mode 100644 index dd311746..00000000 --- a/codex/contracts/interactions/hostinteractions.nim +++ /dev/null @@ -1,24 +0,0 @@ -import pkg/chronos - -import ../../logutils -import ../../sales -import ./interactions - -export sales -export logutils - -type HostInteractions* = ref object of ContractInteractions - sales*: Sales - -proc new*(_: type HostInteractions, clock: Clock, sales: Sales): HostInteractions = - ## Create a new HostInteractions instance - ## - HostInteractions(clock: clock, sales: sales) - -method start*(self: HostInteractions) {.async.} = - await procCall ContractInteractions(self).start() - await self.sales.start() - -method stop*(self: HostInteractions) {.async.} = - await self.sales.stop() - await procCall ContractInteractions(self).start() diff --git a/codex/contracts/interactions/interactions.nim b/codex/contracts/interactions/interactions.nim deleted file mode 100644 index 1006eb3f..00000000 --- a/codex/contracts/interactions/interactions.nim +++ /dev/null @@ -1,15 +0,0 @@ -import pkg/ethers -import ../clock -import ../marketplace -import ../market - -export clock - -type ContractInteractions* = ref object of RootObj - clock*: Clock - -method start*(self: ContractInteractions) {.async, base.} = - discard - -method stop*(self: ContractInteractions) {.async, base.} = - discard diff --git a/codex/contracts/interactions/validatorinteractions.nim b/codex/contracts/interactions/validatorinteractions.nim deleted file mode 100644 index aae28202..00000000 --- a/codex/contracts/interactions/validatorinteractions.nim +++ /dev/null @@ -1,20 +0,0 @@ -import ./interactions -import ../../validation - -export validation - -type ValidatorInteractions* = ref object of ContractInteractions - validation: Validation - -proc new*( - _: type ValidatorInteractions, clock: OnChainClock, validation: Validation -): ValidatorInteractions = - ValidatorInteractions(clock: clock, validation: validation) - -proc start*(self: ValidatorInteractions) {.async.} = - await procCall ContractInteractions(self).start() - await self.validation.start() - -proc stop*(self: ValidatorInteractions) {.async.} = - await self.validation.stop() - await procCall ContractInteractions(self).stop() diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim deleted file mode 100644 index 884441d4..00000000 --- a/codex/contracts/market.nim +++ /dev/null @@ -1,680 +0,0 @@ -import std/strformat -import std/strutils -import pkg/ethers -import pkg/questionable -import pkg/lrucache -import ../utils/exceptions -import ../logutils -import ../market -import ./marketplace -import ./proofs -import ./provider - -export market - -logScope: - topics = "marketplace onchain market" - -type - OnChainMarket* = ref object of Market - contract: Marketplace - signer: Signer - rewardRecipient: ?Address - configuration: ?MarketplaceConfig - requestCache: LruCache[string, StorageRequest] - allowanceLock: AsyncLock - - MarketSubscription = market.Subscription - EventSubscription = ethers.Subscription - OnChainMarketSubscription = ref object of MarketSubscription - eventSubscription: EventSubscription - -func new*( - _: type OnChainMarket, - contract: Marketplace, - rewardRecipient = Address.none, - requestCacheSize: uint16 = DefaultRequestCacheSize, -): OnChainMarket = - without signer =? contract.signer: - raiseAssert("Marketplace contract should have a signer") - - var requestCache = newLruCache[string, StorageRequest](int(requestCacheSize)) - - OnChainMarket( - contract: contract, - signer: signer, - rewardRecipient: rewardRecipient, - requestCache: requestCache, - ) - -proc raiseMarketError(message: string) {.raises: [MarketError].} = - raise newException(MarketError, message) - -func prefixWith(suffix, prefix: string, separator = ": "): string = - if prefix.len > 0: - return &"{prefix}{separator}{suffix}" - else: - return suffix - -template convertEthersError(msg: string = "", body) = - try: - body - except EthersError as error: - raiseMarketError(error.msgDetail.prefixWith(msg)) - -proc config( - market: OnChainMarket -): Future[MarketplaceConfig] {.async: (raises: [CancelledError, MarketError]).} = - without resolvedConfig =? market.configuration: - if err =? (await market.loadConfig()).errorOption: - raiseMarketError(err.msg) - - without config =? market.configuration: - raiseMarketError("Failed to access to config from the Marketplace contract") - - return config - - return resolvedConfig - -template withAllowanceLock*(market: OnChainMarket, body: untyped) = - if market.allowanceLock.isNil: - market.allowanceLock = newAsyncLock() - await market.allowanceLock.acquire() - try: - body - finally: - try: - market.allowanceLock.release() - except AsyncLockError as error: - raise newException(Defect, error.msg, error) - -proc approveFunds( - market: OnChainMarket, amount: UInt256 -) {.async: (raises: [CancelledError, MarketError]).} = - debug "Approving tokens", amount - convertEthersError("Failed to approve funds"): - let tokenAddress = await market.contract.token() - let token = Erc20Token.new(tokenAddress, market.signer) - let owner = await market.signer.getAddress() - let spender = market.contract.address - market.withAllowanceLock: - let allowance = await token.allowance(owner, spender) - discard await token.approve(spender, allowance + amount).confirm(1) - -method loadConfig*( - market: OnChainMarket -): Future[?!void] {.async: (raises: [CancelledError]).} = - try: - without config =? market.configuration: - let fetchedConfig = await market.contract.configuration() - - market.configuration = some fetchedConfig - - return success() - except EthersError as err: - return failure newException( - MarketError, - "Failed to fetch the config from the Marketplace contract: " & err.msg, - ) - -method getZkeyHash*( - market: OnChainMarket -): Future[?string] {.async: (raises: [CancelledError, MarketError]).} = - let config = await market.config() - return some config.proofs.zkeyHash - -method getSigner*( - market: OnChainMarket -): Future[Address] {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to get signer address"): - return await market.signer.getAddress() - -method periodicity*( - market: OnChainMarket -): Future[Periodicity] {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to get Marketplace config"): - let config = await market.config() - let period = config.proofs.period - return Periodicity(seconds: period) - -method proofTimeout*( - market: OnChainMarket -): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to get Marketplace config"): - let config = await market.config() - return config.proofs.timeout - -method repairRewardPercentage*( - market: OnChainMarket -): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to get Marketplace config"): - let config = await market.config() - return config.collateral.repairRewardPercentage - -method requestDurationLimit*(market: OnChainMarket): Future[uint64] {.async.} = - convertEthersError("Failed to get Marketplace config"): - let config = await market.config() - return config.requestDurationLimit - -method proofDowntime*( - market: OnChainMarket -): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to get Marketplace config"): - let config = await market.config() - return config.proofs.downtime - -method getPointer*(market: OnChainMarket, slotId: SlotId): Future[uint8] {.async.} = - convertEthersError("Failed to get slot pointer"): - let overrides = CallOverrides(blockTag: some BlockTag.pending) - return await market.contract.getPointer(slotId, overrides) - -method myRequests*(market: OnChainMarket): Future[seq[RequestId]] {.async.} = - convertEthersError("Failed to get my requests"): - return await market.contract.myRequests - -method mySlots*(market: OnChainMarket): Future[seq[SlotId]] {.async.} = - convertEthersError("Failed to get my slots"): - let slots = await market.contract.mySlots() - debug "Fetched my slots", numSlots = len(slots) - - return slots - -method requestStorage( - market: OnChainMarket, request: StorageRequest -) {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to request storage"): - debug "Requesting storage" - await market.approveFunds(request.totalPrice()) - discard await market.contract.requestStorage(request).confirm(1) - -method getRequest*( - market: OnChainMarket, id: RequestId -): Future[?StorageRequest] {.async: (raises: [CancelledError]).} = - try: - let key = $id - - if key in market.requestCache: - return some market.requestCache[key] - - let request = await market.contract.getRequest(id) - market.requestCache[key] = request - return some request - except Marketplace_UnknownRequest, KeyError: - warn "Cannot retrieve the request", error = getCurrentExceptionMsg() - return none StorageRequest - except EthersError as e: - error "Cannot retrieve the request", error = e.msg - return none StorageRequest - -method requestState*( - market: OnChainMarket, requestId: RequestId -): Future[?RequestState] {.async.} = - convertEthersError("Failed to get request state"): - try: - let overrides = CallOverrides(blockTag: some BlockTag.pending) - return some await market.contract.requestState(requestId, overrides) - except Marketplace_UnknownRequest: - return none RequestState - -method slotState*( - market: OnChainMarket, slotId: SlotId -): Future[SlotState] {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to fetch the slot state from the Marketplace contract"): - let overrides = CallOverrides(blockTag: some BlockTag.pending) - return await market.contract.slotState(slotId, overrides) - -method getRequestEnd*( - market: OnChainMarket, id: RequestId -): Future[SecondsSince1970] {.async.} = - convertEthersError("Failed to get request end"): - return await market.contract.requestEnd(id) - -method requestExpiresAt*( - market: OnChainMarket, id: RequestId -): Future[SecondsSince1970] {.async.} = - convertEthersError("Failed to get request expiry"): - return await market.contract.requestExpiry(id) - -method getHost( - market: OnChainMarket, requestId: RequestId, slotIndex: uint64 -): Future[?Address] {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to get slot's host"): - let slotId = slotId(requestId, slotIndex) - let address = await market.contract.getHost(slotId) - if address != Address.default: - return some address - else: - return none Address - -method currentCollateral*( - market: OnChainMarket, slotId: SlotId -): Future[UInt256] {.async: (raises: [MarketError, CancelledError]).} = - convertEthersError("Failed to get slot's current collateral"): - return await market.contract.currentCollateral(slotId) - -method getActiveSlot*(market: OnChainMarket, slotId: SlotId): Future[?Slot] {.async.} = - convertEthersError("Failed to get active slot"): - try: - return some await market.contract.getActiveSlot(slotId) - except Marketplace_SlotIsFree: - return none Slot - -method fillSlot( - market: OnChainMarket, - requestId: RequestId, - slotIndex: uint64, - proof: Groth16Proof, - collateral: UInt256, -) {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to fill slot"): - logScope: - requestId - slotIndex - - try: - await market.approveFunds(collateral) - - # Add 10% to gas estimate to deal with different evm code flow when we - # happen to be the last one to fill a slot in this request - trace "estimating gas for fillSlot" - let gas = await market.contract.estimateGas.fillSlot(requestId, slotIndex, proof) - let gasLimit = (gas * 110) div 100 - let overrides = TransactionOverrides(gasLimit: some gasLimit) - - trace "calling fillSlot on contract", estimatedGas = gas, gasLimit = gasLimit - discard await market.contract - .fillSlot(requestId, slotIndex, proof, overrides) - .confirm(1) - trace "fillSlot transaction completed" - except Marketplace_SlotNotFree as parent: - raise newException( - SlotStateMismatchError, "Failed to fill slot because the slot is not free", - parent, - ) - -method freeSlot*( - market: OnChainMarket, slotId: SlotId -) {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to free slot"): - try: - var freeSlot: Future[Confirmable] - if rewardRecipient =? market.rewardRecipient: - # If --reward-recipient specified, use it as the reward recipient, and use - # the SP's address as the collateral recipient - let collateralRecipient = await market.getSigner() - - # Add 200% to gas estimate to deal with different evm code flow when we - # happen to be the one to make the request fail - let gas = await market.contract.estimateGas.freeSlot( - slotId, rewardRecipient, collateralRecipient - ) - let gasLimit = gas * 3 - let overrides = TransactionOverrides(gasLimit: some gasLimit) - - trace "calling freeSlot on contract", estimatedGas = gas, gasLimit = gasLimit - - freeSlot = market.contract.freeSlot( - slotId, - rewardRecipient, # --reward-recipient - collateralRecipient, # SP's address - overrides, - ) - else: - # Otherwise, use the SP's address as both the reward and collateral - # recipient (the contract will use msg.sender for both) - - # Add 200% to gas estimate to deal with different evm code flow when we - # happen to be the one to make the request fail - let gas = await market.contract.estimateGas.freeSlot(slotId) - let gasLimit = gas * 3 - let overrides = TransactionOverrides(gasLimit: some (gasLimit)) - - trace "calling freeSlot on contract", estimatedGas = gas, gasLimit = gasLimit - - freeSlot = market.contract.freeSlot(slotId, overrides) - - discard await freeSlot.confirm(1) - except Marketplace_SlotIsFree as parent: - raise newException( - SlotStateMismatchError, "Failed to free slot, slot is already free", parent - ) - -method withdrawFunds( - market: OnChainMarket, requestId: RequestId -) {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to withdraw funds"): - discard await market.contract.withdrawFunds(requestId).confirm(1) - -method isProofRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.} = - convertEthersError("Failed to get proof requirement"): - try: - let overrides = CallOverrides(blockTag: some BlockTag.pending) - return await market.contract.isProofRequired(id, overrides) - except Marketplace_SlotIsFree: - return false - -method willProofBeRequired*(market: OnChainMarket, id: SlotId): Future[bool] {.async.} = - convertEthersError("Failed to get future proof requirement"): - try: - let overrides = CallOverrides(blockTag: some BlockTag.pending) - return await market.contract.willProofBeRequired(id, overrides) - except Marketplace_SlotIsFree: - return false - -method getChallenge*( - market: OnChainMarket, id: SlotId -): Future[ProofChallenge] {.async.} = - convertEthersError("Failed to get proof challenge"): - let overrides = CallOverrides(blockTag: some BlockTag.pending) - return await market.contract.getChallenge(id, overrides) - -method submitProof*( - market: OnChainMarket, id: SlotId, proof: Groth16Proof -) {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to submit proof"): - try: - discard await market.contract.submitProof(id, proof).confirm(1) - except Proofs_InvalidProof as parent: - raise newException( - ProofInvalidError, "Failed to submit proof because the proof is invalid", parent - ) - -method markProofAsMissing*( - market: OnChainMarket, id: SlotId, period: Period -) {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to mark proof as missing"): - # Add 50% to gas estimate to deal with different evm code flow when we - # happen to be the one to make the request fail - let gas = await market.contract.estimateGas.markProofAsMissing(id, period) - let gasLimit = (gas * 150) div 100 - let overrides = TransactionOverrides(gasLimit: some gasLimit) - - trace "calling markProofAsMissing on contract", - estimatedGas = gas, gasLimit = gasLimit - - discard await market.contract.markProofAsMissing(id, period, overrides).confirm(1) - -method canMarkProofAsMissing*( - market: OnChainMarket, id: SlotId, period: Period -): Future[bool] {.async: (raises: [CancelledError]).} = - try: - let overrides = CallOverrides(blockTag: some BlockTag.pending) - discard await market.contract.canMarkProofAsMissing(id, period, overrides) - return true - except EthersError as e: - trace "Proof cannot be marked as missing", msg = e.msg - return false - -method reserveSlot*( - market: OnChainMarket, requestId: RequestId, slotIndex: uint64 -) {.async: (raises: [CancelledError, MarketError]).} = - convertEthersError("Failed to reserve slot"): - try: - # Add 25% to gas estimate to deal with different evm code flow when we - # happen to be the last one that is allowed to reserve the slot - let gas = await market.contract.estimateGas.reserveSlot(requestId, slotIndex) - let gasLimit = (gas * 125) div 100 - let overrides = TransactionOverrides(gasLimit: some gasLimit) - - trace "calling reserveSlot on contract", estimatedGas = gas, gasLimit = gasLimit - - discard - await market.contract.reserveSlot(requestId, slotIndex, overrides).confirm(1) - except SlotReservations_ReservationNotAllowed: - raise newException( - SlotReservationNotAllowedError, - "Failed to reserve slot because reservation is not allowed", - ) - -method canReserveSlot*( - market: OnChainMarket, requestId: RequestId, slotIndex: uint64 -): Future[bool] {.async.} = - convertEthersError("Unable to determine if slot can be reserved"): - return await market.contract.canReserveSlot(requestId, slotIndex) - -method subscribeRequests*( - market: OnChainMarket, callback: OnRequest -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!StorageRequested) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in Request subscription", msg = eventErr.msg - return - - callback(event.requestId, event.ask, event.expiry) - - convertEthersError("Failed to subscribe to StorageRequested events"): - let subscription = await market.contract.subscribe(StorageRequested, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeSlotFilled*( - market: OnChainMarket, callback: OnSlotFilled -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!SlotFilled) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in SlotFilled subscription", msg = eventErr.msg - return - - callback(event.requestId, event.slotIndex) - - convertEthersError("Failed to subscribe to SlotFilled events"): - let subscription = await market.contract.subscribe(SlotFilled, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeSlotFilled*( - market: OnChainMarket, - requestId: RequestId, - slotIndex: uint64, - callback: OnSlotFilled, -): Future[MarketSubscription] {.async.} = - proc onSlotFilled(eventRequestId: RequestId, eventSlotIndex: uint64) = - if eventRequestId == requestId and eventSlotIndex == slotIndex: - callback(requestId, slotIndex) - - convertEthersError("Failed to subscribe to SlotFilled events"): - return await market.subscribeSlotFilled(onSlotFilled) - -method subscribeSlotFreed*( - market: OnChainMarket, callback: OnSlotFreed -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!SlotFreed) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in SlotFreed subscription", msg = eventErr.msg - return - - callback(event.requestId, event.slotIndex) - - convertEthersError("Failed to subscribe to SlotFreed events"): - let subscription = await market.contract.subscribe(SlotFreed, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeSlotReservationsFull*( - market: OnChainMarket, callback: OnSlotReservationsFull -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!SlotReservationsFull) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in SlotReservationsFull subscription", - msg = eventErr.msg - return - - callback(event.requestId, event.slotIndex) - - convertEthersError("Failed to subscribe to SlotReservationsFull events"): - let subscription = await market.contract.subscribe(SlotReservationsFull, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeFulfillment( - market: OnChainMarket, callback: OnFulfillment -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestFulfilled) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in RequestFulfillment subscription", msg = eventErr.msg - return - - callback(event.requestId) - - convertEthersError("Failed to subscribe to RequestFulfilled events"): - let subscription = await market.contract.subscribe(RequestFulfilled, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeFulfillment( - market: OnChainMarket, requestId: RequestId, callback: OnFulfillment -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestFulfilled) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in RequestFulfillment subscription", msg = eventErr.msg - return - - if event.requestId == requestId: - callback(event.requestId) - - convertEthersError("Failed to subscribe to RequestFulfilled events"): - let subscription = await market.contract.subscribe(RequestFulfilled, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeRequestCancelled*( - market: OnChainMarket, callback: OnRequestCancelled -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestCancelled) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in RequestCancelled subscription", msg = eventErr.msg - return - - callback(event.requestId) - - convertEthersError("Failed to subscribe to RequestCancelled events"): - let subscription = await market.contract.subscribe(RequestCancelled, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeRequestCancelled*( - market: OnChainMarket, requestId: RequestId, callback: OnRequestCancelled -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestCancelled) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in RequestCancelled subscription", msg = eventErr.msg - return - - if event.requestId == requestId: - callback(event.requestId) - - convertEthersError("Failed to subscribe to RequestCancelled events"): - let subscription = await market.contract.subscribe(RequestCancelled, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeRequestFailed*( - market: OnChainMarket, callback: OnRequestFailed -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestFailed) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in RequestFailed subscription", msg = eventErr.msg - return - - callback(event.requestId) - - convertEthersError("Failed to subscribe to RequestFailed events"): - let subscription = await market.contract.subscribe(RequestFailed, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeRequestFailed*( - market: OnChainMarket, requestId: RequestId, callback: OnRequestFailed -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!RequestFailed) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in RequestFailed subscription", msg = eventErr.msg - return - - if event.requestId == requestId: - callback(event.requestId) - - convertEthersError("Failed to subscribe to RequestFailed events"): - let subscription = await market.contract.subscribe(RequestFailed, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method subscribeProofSubmission*( - market: OnChainMarket, callback: OnProofSubmitted -): Future[MarketSubscription] {.async.} = - proc onEvent(eventResult: ?!ProofSubmitted) {.raises: [].} = - without event =? eventResult, eventErr: - error "There was an error in ProofSubmitted subscription", msg = eventErr.msg - return - - callback(event.id) - - convertEthersError("Failed to subscribe to ProofSubmitted events"): - let subscription = await market.contract.subscribe(ProofSubmitted, onEvent) - return OnChainMarketSubscription(eventSubscription: subscription) - -method unsubscribe*(subscription: OnChainMarketSubscription) {.async.} = - await subscription.eventSubscription.unsubscribe() - -method queryPastSlotFilledEvents*( - market: OnChainMarket, fromBlock: BlockTag -): Future[seq[SlotFilled]] {.async.} = - convertEthersError("Failed to get past SlotFilled events from block"): - return await market.contract.queryFilter(SlotFilled, fromBlock, BlockTag.latest) - -method queryPastSlotFilledEvents*( - market: OnChainMarket, blocksAgo: int -): Future[seq[SlotFilled]] {.async.} = - convertEthersError("Failed to get past SlotFilled events"): - let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo) - - return await market.queryPastSlotFilledEvents(fromBlock) - -method queryPastSlotFilledEvents*( - market: OnChainMarket, fromTime: SecondsSince1970 -): Future[seq[SlotFilled]] {.async.} = - convertEthersError("Failed to get past SlotFilled events from time"): - let fromBlock = await market.contract.provider.blockNumberForEpoch(fromTime) - return await market.queryPastSlotFilledEvents(BlockTag.init(fromBlock)) - -method queryPastStorageRequestedEvents*( - market: OnChainMarket, fromBlock: BlockTag -): Future[seq[StorageRequested]] {.async.} = - convertEthersError("Failed to get past StorageRequested events from block"): - return - await market.contract.queryFilter(StorageRequested, fromBlock, BlockTag.latest) - -method queryPastStorageRequestedEvents*( - market: OnChainMarket, blocksAgo: int -): Future[seq[StorageRequested]] {.async.} = - convertEthersError("Failed to get past StorageRequested events"): - let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo) - - return await market.queryPastStorageRequestedEvents(fromBlock) - -method slotCollateral*( - market: OnChainMarket, requestId: RequestId, slotIndex: uint64 -): Future[?!UInt256] {.async: (raises: [CancelledError]).} = - let slotid = slotId(requestId, slotIndex) - - try: - let slotState = await market.slotState(slotid) - - without request =? await market.getRequest(requestId): - return failure newException( - MarketError, "Failure calculating the slotCollateral, cannot get the request" - ) - - return market.slotCollateral(request.ask.collateralPerSlot, slotState) - except MarketError as error: - error "Error when trying to calculate the slotCollateral", error = error.msg - return failure error - -method slotCollateral*( - market: OnChainMarket, collateralPerSlot: UInt256, slotState: SlotState -): ?!UInt256 {.raises: [].} = - if slotState == SlotState.Repair: - without repairRewardPercentage =? - market.configuration .? collateral .? repairRewardPercentage: - return failure newException( - MarketError, - "Failure calculating the slotCollateral, cannot get the reward percentage", - ) - - return success ( - collateralPerSlot - (collateralPerSlot * repairRewardPercentage.u256).div( - 100.u256 - ) - ) - - return success(collateralPerSlot) diff --git a/codex/contracts/marketplace.nim b/codex/contracts/marketplace.nim deleted file mode 100644 index 95de3dcf..00000000 --- a/codex/contracts/marketplace.nim +++ /dev/null @@ -1,198 +0,0 @@ -import pkg/ethers -import pkg/ethers/erc20 -import pkg/json_rpc/rpcclient -import pkg/stint -import pkg/chronos -import ../clock -import ./requests -import ./proofs -import ./config - -export stint -export ethers except `%`, `%*`, toJson -export erc20 except `%`, `%*`, toJson -export config -export requests - -type - Marketplace* = ref object of Contract - - Marketplace_RepairRewardPercentageTooHigh* = object of SolidityError - Marketplace_SlashPercentageTooHigh* = object of SolidityError - Marketplace_MaximumSlashingTooHigh* = object of SolidityError - Marketplace_InvalidExpiry* = object of SolidityError - Marketplace_InvalidMaxSlotLoss* = object of SolidityError - Marketplace_InsufficientSlots* = object of SolidityError - Marketplace_InvalidClientAddress* = object of SolidityError - Marketplace_RequestAlreadyExists* = object of SolidityError - Marketplace_InvalidSlot* = object of SolidityError - Marketplace_SlotNotFree* = object of SolidityError - Marketplace_InvalidSlotHost* = object of SolidityError - Marketplace_AlreadyPaid* = object of SolidityError - Marketplace_TransferFailed* = object of SolidityError - Marketplace_UnknownRequest* = object of SolidityError - Marketplace_InvalidState* = object of SolidityError - Marketplace_StartNotBeforeExpiry* = object of SolidityError - Marketplace_SlotNotAcceptingProofs* = object of SolidityError - Marketplace_SlotIsFree* = object of SolidityError - Marketplace_ReservationRequired* = object of SolidityError - Marketplace_NothingToWithdraw* = object of SolidityError - Marketplace_InsufficientDuration* = object of SolidityError - Marketplace_InsufficientProofProbability* = object of SolidityError - Marketplace_InsufficientCollateral* = object of SolidityError - Marketplace_InsufficientReward* = object of SolidityError - Marketplace_InvalidCid* = object of SolidityError - Marketplace_DurationExceedsLimit* = object of SolidityError - Proofs_InsufficientBlockHeight* = object of SolidityError - Proofs_InvalidProof* = object of SolidityError - Proofs_ProofAlreadySubmitted* = object of SolidityError - Proofs_PeriodNotEnded* = object of SolidityError - Proofs_ValidationTimedOut* = object of SolidityError - Proofs_ProofNotMissing* = object of SolidityError - Proofs_ProofNotRequired* = object of SolidityError - Proofs_ProofAlreadyMarkedMissing* = object of SolidityError - Periods_InvalidSecondsPerPeriod* = object of SolidityError - SlotReservations_ReservationNotAllowed* = object of SolidityError - -proc configuration*(marketplace: Marketplace): MarketplaceConfig {.contract, view.} -proc token*(marketplace: Marketplace): Address {.contract, view.} -proc currentCollateral*( - marketplace: Marketplace, id: SlotId -): UInt256 {.contract, view.} - -proc requestStorage*( - marketplace: Marketplace, request: StorageRequest -): Confirmable {. - contract, - errors: [ - Marketplace_InvalidClientAddress, Marketplace_RequestAlreadyExists, - Marketplace_InvalidExpiry, Marketplace_InsufficientSlots, - Marketplace_InvalidMaxSlotLoss, Marketplace_InsufficientDuration, - Marketplace_InsufficientProofProbability, Marketplace_InsufficientCollateral, - Marketplace_InsufficientReward, Marketplace_InvalidCid, - ] -.} - -proc fillSlot*( - marketplace: Marketplace, requestId: RequestId, slotIndex: uint64, proof: Groth16Proof -): Confirmable {. - contract, - errors: [ - Marketplace_InvalidSlot, Marketplace_ReservationRequired, Marketplace_SlotNotFree, - Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, - ] -.} - -proc withdrawFunds*( - marketplace: Marketplace, requestId: RequestId -): Confirmable {. - contract, - errors: [ - Marketplace_InvalidClientAddress, Marketplace_InvalidState, - Marketplace_NothingToWithdraw, Marketplace_UnknownRequest, - ] -.} - -proc withdrawFunds*( - marketplace: Marketplace, requestId: RequestId, withdrawAddress: Address -): Confirmable {. - contract, - errors: [ - Marketplace_InvalidClientAddress, Marketplace_InvalidState, - Marketplace_NothingToWithdraw, Marketplace_UnknownRequest, - ] -.} - -proc freeSlot*( - marketplace: Marketplace, id: SlotId -): Confirmable {. - contract, - errors: [ - Marketplace_InvalidSlotHost, Marketplace_AlreadyPaid, - Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, Marketplace_SlotIsFree, - ] -.} - -proc freeSlot*( - marketplace: Marketplace, - id: SlotId, - rewardRecipient: Address, - collateralRecipient: Address, -): Confirmable {. - contract, - errors: [ - Marketplace_InvalidSlotHost, Marketplace_AlreadyPaid, - Marketplace_StartNotBeforeExpiry, Marketplace_UnknownRequest, Marketplace_SlotIsFree, - ] -.} - -proc getRequest*( - marketplace: Marketplace, id: RequestId -): StorageRequest {.contract, view, errors: [Marketplace_UnknownRequest].} - -proc getHost*(marketplace: Marketplace, id: SlotId): Address {.contract, view.} -proc getActiveSlot*( - marketplace: Marketplace, id: SlotId -): Slot {.contract, view, errors: [Marketplace_SlotIsFree].} - -proc myRequests*(marketplace: Marketplace): seq[RequestId] {.contract, view.} -proc mySlots*(marketplace: Marketplace): seq[SlotId] {.contract, view.} -proc requestState*( - marketplace: Marketplace, requestId: RequestId -): RequestState {.contract, view, errors: [Marketplace_UnknownRequest].} - -proc slotState*(marketplace: Marketplace, slotId: SlotId): SlotState {.contract, view.} -proc requestEnd*( - marketplace: Marketplace, requestId: RequestId -): SecondsSince1970 {.contract, view.} - -proc requestExpiry*( - marketplace: Marketplace, requestId: RequestId -): SecondsSince1970 {.contract, view.} - -proc missingProofs*(marketplace: Marketplace, id: SlotId): UInt256 {.contract, view.} -proc isProofRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.} -proc willProofBeRequired*(marketplace: Marketplace, id: SlotId): bool {.contract, view.} -proc getChallenge*( - marketplace: Marketplace, id: SlotId -): array[32, byte] {.contract, view.} - -proc getPointer*(marketplace: Marketplace, id: SlotId): uint8 {.contract, view.} - -proc submitProof*( - marketplace: Marketplace, id: SlotId, proof: Groth16Proof -): Confirmable {. - contract, - errors: - [Proofs_ProofAlreadySubmitted, Proofs_InvalidProof, Marketplace_UnknownRequest] -.} - -proc markProofAsMissing*( - marketplace: Marketplace, id: SlotId, period: uint64 -): Confirmable {. - contract, - errors: [ - Marketplace_SlotNotAcceptingProofs, Marketplace_StartNotBeforeExpiry, - Proofs_PeriodNotEnded, Proofs_ValidationTimedOut, Proofs_ProofNotMissing, - Proofs_ProofNotRequired, Proofs_ProofAlreadyMarkedMissing, - ] -.} - -proc canMarkProofAsMissing*( - marketplace: Marketplace, id: SlotId, period: uint64 -): Confirmable {. - contract, - errors: [ - Marketplace_SlotNotAcceptingProofs, Proofs_PeriodNotEnded, - Proofs_ValidationTimedOut, Proofs_ProofNotMissing, Proofs_ProofNotRequired, - Proofs_ProofAlreadyMarkedMissing, - ] -.} - -proc reserveSlot*( - marketplace: Marketplace, requestId: RequestId, slotIndex: uint64 -): Confirmable {.contract.} - -proc canReserveSlot*( - marketplace: Marketplace, requestId: RequestId, slotIndex: uint64 -): bool {.contract, view.} diff --git a/codex/contracts/proofs.nim b/codex/contracts/proofs.nim deleted file mode 100644 index c0d80b7d..00000000 --- a/codex/contracts/proofs.nim +++ /dev/null @@ -1,46 +0,0 @@ -import pkg/stint -import pkg/contractabi -import pkg/ethers/contracts/fields - -type - Groth16Proof* = object - a*: G1Point - b*: G2Point - c*: G1Point - - G1Point* = object - x*: UInt256 - y*: UInt256 - - # A field element F_{p^2} encoded as `real + i * imag` - Fp2Element* = object - real*: UInt256 - imag*: UInt256 - - G2Point* = object - x*: Fp2Element - y*: Fp2Element - -func solidityType*(_: type G1Point): string = - solidityType(G1Point.fieldTypes) - -func solidityType*(_: type Fp2Element): string = - solidityType(Fp2Element.fieldTypes) - -func solidityType*(_: type G2Point): string = - solidityType(G2Point.fieldTypes) - -func solidityType*(_: type Groth16Proof): string = - solidityType(Groth16Proof.fieldTypes) - -func encode*(encoder: var AbiEncoder, point: G1Point) = - encoder.write(point.fieldValues) - -func encode*(encoder: var AbiEncoder, element: Fp2Element) = - encoder.write(element.fieldValues) - -func encode*(encoder: var AbiEncoder, point: G2Point) = - encoder.write(point.fieldValues) - -func encode*(encoder: var AbiEncoder, proof: Groth16Proof) = - encoder.write(proof.fieldValues) diff --git a/codex/contracts/provider.nim b/codex/contracts/provider.nim deleted file mode 100644 index b1576bb0..00000000 --- a/codex/contracts/provider.nim +++ /dev/null @@ -1,123 +0,0 @@ -import pkg/ethers/provider -import pkg/chronos -import pkg/questionable - -import ../logutils - -from ../clock import SecondsSince1970 - -logScope: - topics = "marketplace onchain provider" - -proc raiseProviderError(message: string) {.raises: [ProviderError].} = - raise newException(ProviderError, message) - -proc blockNumberAndTimestamp*( - provider: Provider, blockTag: BlockTag -): Future[(UInt256, UInt256)] {.async: (raises: [ProviderError, CancelledError]).} = - without latestBlock =? await provider.getBlock(blockTag): - raiseProviderError("Could not get latest block") - - without latestBlockNumber =? latestBlock.number: - raiseProviderError("Could not get latest block number") - - return (latestBlockNumber, latestBlock.timestamp) - -proc binarySearchFindClosestBlock( - provider: Provider, epochTime: int, low: UInt256, high: UInt256 -): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} = - let (_, lowTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(low)) - let (_, highTimestamp) = await provider.blockNumberAndTimestamp(BlockTag.init(high)) - if abs(lowTimestamp.truncate(int) - epochTime) < - abs(highTimestamp.truncate(int) - epochTime): - return low - else: - return high - -proc binarySearchBlockNumberForEpoch( - provider: Provider, - epochTime: UInt256, - latestBlockNumber: UInt256, - earliestBlockNumber: UInt256, -): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} = - var low = earliestBlockNumber - var high = latestBlockNumber - - while low <= high: - if low == 0 and high == 0: - return low - let mid = (low + high) div 2 - let (midBlockNumber, midBlockTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.init(mid)) - - if midBlockTimestamp < epochTime: - low = mid + 1 - elif midBlockTimestamp > epochTime: - high = mid - 1 - else: - return midBlockNumber - # NOTICE that by how the binary search is implemented, when it finishes - # low is always greater than high - this is why we use high, where - # intuitively we would use low: - await provider.binarySearchFindClosestBlock( - epochTime.truncate(int), low = high, high = low - ) - -proc blockNumberForEpoch*( - provider: Provider, epochTime: SecondsSince1970 -): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} = - let epochTimeUInt256 = epochTime.u256 - let (latestBlockNumber, latestBlockTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.latest) - let (earliestBlockNumber, earliestBlockTimestamp) = - await provider.blockNumberAndTimestamp(BlockTag.earliest) - - # Initially we used the average block time to predict - # the number of blocks we need to look back in order to find - # the block number corresponding to the given epoch time. - # This estimation can be highly inaccurate if block time - # was changing in the past or is fluctuating and therefore - # we used that information initially only to find out - # if the available history is long enough to perform effective search. - # It turns out we do not have to do that. There is an easier way. - # - # First we check if the given epoch time equals the timestamp of either - # the earliest or the latest block. If it does, we just return the - # block number of that block. - # - # Otherwise, if the earliest available block is not the genesis block, - # we should check the timestamp of that earliest block and if it is greater - # than the epoch time, we should issue a warning and return - # that earliest block number. - # In all other cases, thus when the earliest block is not the genesis - # block but its timestamp is not greater than the requested epoch time, or - # if the earliest available block is the genesis block, - # (which means we have the whole history available), we should proceed with - # the binary search. - # - # Additional benefit of this method is that we do not have to rely - # on the average block time, which not only makes the whole thing - # more reliable, but also easier to test. - - # Are lucky today? - if earliestBlockTimestamp == epochTimeUInt256: - return earliestBlockNumber - if latestBlockTimestamp == epochTimeUInt256: - return latestBlockNumber - - if earliestBlockNumber > 0 and earliestBlockTimestamp > epochTimeUInt256: - let availableHistoryInDays = - (latestBlockTimestamp - earliestBlockTimestamp) div 1.days.secs.u256 - warn "Short block history detected.", - earliestBlockTimestamp = earliestBlockTimestamp, days = availableHistoryInDays - return earliestBlockNumber - - return await provider.binarySearchBlockNumberForEpoch( - epochTimeUInt256, latestBlockNumber, earliestBlockNumber - ) - -proc pastBlockTag*( - provider: Provider, blocksAgo: int -): Future[BlockTag] {.async: (raises: [ProviderError, CancelledError]).} = - let head = await provider.getBlockNumber() - return BlockTag.init(head - blocksAgo.abs.u256) diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim deleted file mode 100644 index f6f630e2..00000000 --- a/codex/contracts/requests.nim +++ /dev/null @@ -1,206 +0,0 @@ -import std/hashes -import std/sequtils -import std/typetraits -import pkg/contractabi -import pkg/nimcrypto/keccak -import pkg/ethers/contracts/fields -import pkg/questionable/results -import pkg/stew/byteutils -import pkg/libp2p/[cid, multicodec] -import ../logutils -import ../utils/json -from ../errors import mapFailure - -export contractabi - -type - StorageRequest* = object - client* {.serialize.}: Address - ask* {.serialize.}: StorageAsk - content* {.serialize.}: StorageContent - expiry* {.serialize.}: uint64 - nonce*: Nonce - - StorageAsk* = object - proofProbability* {.serialize.}: UInt256 - pricePerBytePerSecond* {.serialize.}: UInt256 - collateralPerByte* {.serialize.}: UInt256 - slots* {.serialize.}: uint64 - slotSize* {.serialize.}: uint64 - duration* {.serialize.}: uint64 - maxSlotLoss* {.serialize.}: uint64 - - StorageContent* = object - cid* {.serialize.}: Cid - merkleRoot*: array[32, byte] - - Slot* = object - request* {.serialize.}: StorageRequest - slotIndex* {.serialize.}: uint64 - - SlotId* = distinct array[32, byte] - RequestId* = distinct array[32, byte] - Nonce* = distinct array[32, byte] - RequestState* {.pure.} = enum - New - Started - Cancelled - Finished - Failed - - SlotState* {.pure.} = enum - Free - Filled - Finished - Failed - Paid - Cancelled - Repair - -proc `==`*(x, y: Nonce): bool {.borrow.} -proc `==`*(x, y: RequestId): bool {.borrow.} -proc `==`*(x, y: SlotId): bool {.borrow.} -proc hash*(x: SlotId): Hash {.borrow.} -proc hash*(x: Nonce): Hash {.borrow.} -proc hash*(x: Address): Hash {.borrow.} - -func toArray*(id: RequestId | SlotId | Nonce): array[32, byte] = - array[32, byte](id) - -proc `$`*(id: RequestId | SlotId | Nonce): string = - id.toArray.toHex - -proc fromHex*(T: type RequestId, hex: string): T = - T array[32, byte].fromHex(hex) - -proc fromHex*(T: type SlotId, hex: string): T = - T array[32, byte].fromHex(hex) - -proc fromHex*(T: type Nonce, hex: string): T = - T array[32, byte].fromHex(hex) - -proc fromHex*[T: distinct](_: type T, hex: string): T = - type baseType = T.distinctBase - T baseType.fromHex(hex) - -proc toHex*[T: distinct](id: T): string = - type baseType = T.distinctBase - baseType(id).toHex - -logutils.formatIt(LogFormat.textLines, Nonce): - it.short0xHexLog -logutils.formatIt(LogFormat.textLines, RequestId): - it.short0xHexLog -logutils.formatIt(LogFormat.textLines, SlotId): - it.short0xHexLog -logutils.formatIt(LogFormat.json, Nonce): - it.to0xHexLog -logutils.formatIt(LogFormat.json, RequestId): - it.to0xHexLog -logutils.formatIt(LogFormat.json, SlotId): - it.to0xHexLog - -func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest = - StorageRequest( - client: tupl[0], ask: tupl[1], content: tupl[2], expiry: tupl[3], nonce: tupl[4] - ) - -func fromTuple(_: type Slot, tupl: tuple): Slot = - Slot(request: tupl[0], slotIndex: tupl[1]) - -func fromTuple(_: type StorageAsk, tupl: tuple): StorageAsk = - StorageAsk( - proofProbability: tupl[0], - pricePerBytePerSecond: tupl[1], - collateralPerByte: tupl[2], - slots: tupl[3], - slotSize: tupl[4], - duration: tupl[5], - maxSlotLoss: tupl[6], - ) - -func fromTuple(_: type StorageContent, tupl: tuple): StorageContent = - StorageContent(cid: tupl[0], merkleRoot: tupl[1]) - -func solidityType*(_: type Cid): string = - solidityType(seq[byte]) - -func solidityType*(_: type StorageContent): string = - solidityType(StorageContent.fieldTypes) - -func solidityType*(_: type StorageAsk): string = - solidityType(StorageAsk.fieldTypes) - -func solidityType*(_: type StorageRequest): string = - solidityType(StorageRequest.fieldTypes) - -# Note: it seems to be ok to ignore the vbuffer offset for now -func encode*(encoder: var AbiEncoder, cid: Cid) = - encoder.write(cid.data.buffer) - -func encode*(encoder: var AbiEncoder, content: StorageContent) = - encoder.write(content.fieldValues) - -func encode*(encoder: var AbiEncoder, ask: StorageAsk) = - encoder.write(ask.fieldValues) - -func encode*(encoder: var AbiEncoder, id: RequestId | SlotId | Nonce) = - encoder.write(id.toArray) - -func encode*(encoder: var AbiEncoder, request: StorageRequest) = - encoder.write(request.fieldValues) - -func encode*(encoder: var AbiEncoder, slot: Slot) = - encoder.write(slot.fieldValues) - -func decode*(decoder: var AbiDecoder, T: type Cid): ?!T = - let data = ?decoder.read(seq[byte]) - Cid.init(data).mapFailure - -func decode*(decoder: var AbiDecoder, T: type StorageContent): ?!T = - let tupl = ?decoder.read(StorageContent.fieldTypes) - success StorageContent.fromTuple(tupl) - -func decode*(decoder: var AbiDecoder, T: type StorageAsk): ?!T = - let tupl = ?decoder.read(StorageAsk.fieldTypes) - success StorageAsk.fromTuple(tupl) - -func decode*(decoder: var AbiDecoder, T: type StorageRequest): ?!T = - let tupl = ?decoder.read(StorageRequest.fieldTypes) - success StorageRequest.fromTuple(tupl) - -func decode*(decoder: var AbiDecoder, T: type Slot): ?!T = - let tupl = ?decoder.read(Slot.fieldTypes) - success Slot.fromTuple(tupl) - -func id*(request: StorageRequest): RequestId = - let encoding = AbiEncoder.encode((request,)) - RequestId(keccak256.digest(encoding).data) - -func slotId*(requestId: RequestId, slotIndex: uint64): SlotId = - let encoding = AbiEncoder.encode((requestId, slotIndex)) - SlotId(keccak256.digest(encoding).data) - -func slotId*(request: StorageRequest, slotIndex: uint64): SlotId = - slotId(request.id, slotIndex) - -func id*(slot: Slot): SlotId = - slotId(slot.request, slot.slotIndex) - -func pricePerSlotPerSecond*(ask: StorageAsk): UInt256 = - ask.pricePerBytePerSecond * ask.slotSize.u256 - -func pricePerSlot*(ask: StorageAsk): UInt256 = - ask.duration.u256 * ask.pricePerSlotPerSecond - -func totalPrice*(ask: StorageAsk): UInt256 = - ask.slots.u256 * ask.pricePerSlot - -func totalPrice*(request: StorageRequest): UInt256 = - request.ask.totalPrice - -func collateralPerSlot*(ask: StorageAsk): UInt256 = - ask.collateralPerByte * ask.slotSize.u256 - -func size*(ask: StorageAsk): uint64 = - ask.slots * ask.slotSize diff --git a/codex/erasure.nim b/codex/erasure.nim deleted file mode 100644 index f9310a40..00000000 --- a/codex/erasure.nim +++ /dev/null @@ -1,25 +0,0 @@ -## Logos Storage -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import ./erasure/erasure -import ./erasure/backends/leopard - -export erasure - -func leoEncoderProvider*( - size, buffers, parity: int -): EncoderBackend {.raises: [Defect].} = - ## create new Leo Encoder - LeoEncoderBackend.new(size, buffers, parity) - -func leoDecoderProvider*( - size, buffers, parity: int -): DecoderBackend {.raises: [Defect].} = - ## create new Leo Decoder - LeoDecoderBackend.new(size, buffers, parity) diff --git a/codex/erasure/backend.nim b/codex/erasure/backend.nim deleted file mode 100644 index 9885326f..00000000 --- a/codex/erasure/backend.nim +++ /dev/null @@ -1,44 +0,0 @@ -## Logos Storage -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [], gcsafe.} - -import ../stores - -type - ErasureBackend* = ref object of RootObj - blockSize*: int # block size in bytes - buffers*: int # number of original pieces - parity*: int # number of redundancy pieces - - EncoderBackend* = ref object of ErasureBackend - DecoderBackend* = ref object of ErasureBackend - -method release*(self: ErasureBackend) {.base, gcsafe.} = - ## release the backend - ## - raiseAssert("not implemented!") - -method encode*( - self: EncoderBackend, - buffers, parity: ptr UncheckedArray[ptr UncheckedArray[byte]], - dataLen, parityLen: int, -): Result[void, cstring] {.base, gcsafe.} = - ## encode buffers using a backend - ## - raiseAssert("not implemented!") - -method decode*( - self: DecoderBackend, - buffers, parity, recovered: ptr UncheckedArray[ptr UncheckedArray[byte]], - dataLen, parityLen, recoveredLen: int, -): Result[void, cstring] {.base, gcsafe.} = - ## decode buffers using a backend - ## - raiseAssert("not implemented!") diff --git a/codex/erasure/backends/leopard.nim b/codex/erasure/backends/leopard.nim deleted file mode 100644 index 66bc059d..00000000 --- a/codex/erasure/backends/leopard.nim +++ /dev/null @@ -1,79 +0,0 @@ -## Logos Storage -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/options - -import pkg/leopard -import pkg/results - -import ../backend - -type - LeoEncoderBackend* = ref object of EncoderBackend - encoder*: Option[LeoEncoder] - - LeoDecoderBackend* = ref object of DecoderBackend - decoder*: Option[LeoDecoder] - -method encode*( - self: LeoEncoderBackend, - data, parity: ptr UncheckedArray[ptr UncheckedArray[byte]], - dataLen, parityLen: int, -): Result[void, cstring] = - ## Encode data using Leopard backend - - if parityLen == 0: - return ok() - - var encoder = - if self.encoder.isNone: - self.encoder = (?LeoEncoder.init(self.blockSize, self.buffers, self.parity)).some - self.encoder.get() - else: - self.encoder.get() - - encoder.encode(data, parity, dataLen, parityLen) - -method decode*( - self: LeoDecoderBackend, - data, parity, recovered: ptr UncheckedArray[ptr UncheckedArray[byte]], - dataLen, parityLen, recoveredLen: int, -): Result[void, cstring] = - ## Decode data using given Leopard backend - - var decoder = - if self.decoder.isNone: - self.decoder = (?LeoDecoder.init(self.blockSize, self.buffers, self.parity)).some - self.decoder.get() - else: - self.decoder.get() - - decoder.decode(data, parity, recovered, dataLen, parityLen, recoveredLen) - -method release*(self: LeoEncoderBackend) = - if self.encoder.isSome: - self.encoder.get().free() - -method release*(self: LeoDecoderBackend) = - if self.decoder.isSome: - self.decoder.get().free() - -proc new*( - T: type LeoEncoderBackend, blockSize, buffers, parity: int -): LeoEncoderBackend = - ## Create an instance of an Leopard Encoder backend - ## - LeoEncoderBackend(blockSize: blockSize, buffers: buffers, parity: parity) - -proc new*( - T: type LeoDecoderBackend, blockSize, buffers, parity: int -): LeoDecoderBackend = - ## Create an instance of an Leopard Decoder backend - ## - LeoDecoderBackend(blockSize: blockSize, buffers: buffers, parity: parity) diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim deleted file mode 100644 index c8afe440..00000000 --- a/codex/erasure/erasure.nim +++ /dev/null @@ -1,728 +0,0 @@ -## Logos Storage -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [], gcsafe.} - -import std/[sugar, atomics, sequtils] - -import pkg/chronos -import pkg/chronos/threadsync -import pkg/chronicles -import pkg/libp2p/[multicodec, cid, multihash] -import pkg/libp2p/protobuf/minprotobuf -import pkg/taskpools - -import ../logutils -import ../manifest -import ../merkletree -import ../stores -import ../clock -import ../blocktype as bt -import ../utils -import ../utils/asynciter -import ../indexingstrategy -import ../errors -import ../utils/arrayutils - -import pkg/stew/byteutils - -import ./backend - -export backend - -logScope: - topics = "codex erasure" - -type - ## Encode a manifest into one that is erasure protected. - ## - ## The new manifest has K `blocks` that are encoded into - ## additional M `parity` blocks. The resulting dataset - ## is padded with empty blocks if it doesn't have a square - ## shape. - ## - ## NOTE: The padding blocks could be excluded - ## from transmission, but they aren't for now. - ## - ## The resulting dataset is logically divided into rows - ## where a row is made up of B blocks. There are then, - ## K + M = N rows in total, each of length B blocks. Rows - ## are assumed to be of the same number of (B) blocks. - ## - ## The encoding is systematic and the rows can be - ## read sequentially by any node without decoding. - ## - ## Decoding is possible with any K rows or partial K - ## columns (with up to M blocks missing per column), - ## or any combination there of. - ## - EncoderProvider* = - proc(size, blocks, parity: int): EncoderBackend {.raises: [Defect], noSideEffect.} - - DecoderProvider* = - proc(size, blocks, parity: int): DecoderBackend {.raises: [Defect], noSideEffect.} - - Erasure* = ref object - taskPool: Taskpool - encoderProvider*: EncoderProvider - decoderProvider*: DecoderProvider - store*: BlockStore - - EncodingParams = object - ecK: Natural - ecM: Natural - rounded: Natural - steps: Natural - blocksCount: Natural - strategy: StrategyType - - ErasureError* = object of CodexError - InsufficientBlocksError* = object of ErasureError - # Minimum size, in bytes, that the dataset must have had - # for the encoding request to have succeeded with the parameters - # provided. - minSize*: NBytes - - EncodeTask = object - success: Atomic[bool] - erasure: ptr Erasure - blocks: ptr UncheckedArray[ptr UncheckedArray[byte]] - parity: ptr UncheckedArray[ptr UncheckedArray[byte]] - blockSize, blocksLen, parityLen: int - signal: ThreadSignalPtr - - DecodeTask = object - success: Atomic[bool] - erasure: ptr Erasure - blocks: ptr UncheckedArray[ptr UncheckedArray[byte]] - parity: ptr UncheckedArray[ptr UncheckedArray[byte]] - recovered: ptr UncheckedArray[ptr UncheckedArray[byte]] - blockSize, blocksLen: int - parityLen, recoveredLen: int - signal: ThreadSignalPtr - -func indexToPos(steps, idx, step: int): int {.inline.} = - ## Convert an index to a position in the encoded - ## dataset - ## `idx` - the index to convert - ## `step` - the current step - ## `pos` - the position in the encoded dataset - ## - - (idx - step) div steps - -proc getPendingBlocks( - self: Erasure, manifest: Manifest, indices: seq[int] -): AsyncIter[(?!bt.Block, int)] = - ## Get pending blocks iterator - ## - var pendingBlocks: seq[Future[(?!bt.Block, int)]] = @[] - - proc attachIndex( - fut: Future[?!bt.Block], i: int - ): Future[(?!bt.Block, int)] {.async.} = - ## avoids closure capture issues - return (await fut, i) - - for blockIndex in indices: - # request blocks from the store - let fut = self.store.getBlock(BlockAddress.init(manifest.treeCid, blockIndex)) - pendingBlocks.add(attachIndex(fut, blockIndex)) - - proc isFinished(): bool = - pendingBlocks.len == 0 - - proc genNext(): Future[(?!bt.Block, int)] {.async.} = - let completedFut = await one(pendingBlocks) - if (let i = pendingBlocks.find(completedFut); i >= 0): - pendingBlocks.del(i) - return await completedFut - else: - let (_, index) = await completedFut - raise newException( - CatchableError, - "Future for block id not found, tree cid: " & $manifest.treeCid & ", index: " & - $index, - ) - - AsyncIter[(?!bt.Block, int)].new(genNext, isFinished) - -proc prepareEncodingData( - self: Erasure, - manifest: Manifest, - params: EncodingParams, - step: Natural, - data: ref seq[seq[byte]], - cids: ref seq[Cid], - emptyBlock: seq[byte], -): Future[?!Natural] {.async.} = - ## Prepare data for encoding - ## - - let - strategy = params.strategy.init( - firstIndex = 0, lastIndex = params.rounded - 1, iterations = params.steps - ) - indices = toSeq(strategy.getIndices(step)) - pendingBlocksIter = - self.getPendingBlocks(manifest, indices.filterIt(it < manifest.blocksCount)) - - var resolved = 0 - for fut in pendingBlocksIter: - let (blkOrErr, idx) = await fut - without blk =? blkOrErr, err: - warn "Failed retrieving a block", treeCid = manifest.treeCid, idx, msg = err.msg - return failure(err) - - let pos = indexToPos(params.steps, idx, step) - shallowCopy(data[pos], if blk.isEmpty: emptyBlock else: blk.data) - cids[idx] = blk.cid - - resolved.inc() - - for idx in indices.filterIt(it >= manifest.blocksCount): - let pos = indexToPos(params.steps, idx, step) - trace "Padding with empty block", idx - shallowCopy(data[pos], emptyBlock) - without emptyBlockCid =? emptyCid(manifest.version, manifest.hcodec, manifest.codec), - err: - return failure(err) - cids[idx] = emptyBlockCid - - success(resolved.Natural) - -proc prepareDecodingData( - self: Erasure, - encoded: Manifest, - step: Natural, - data: ref seq[seq[byte]], - parityData: ref seq[seq[byte]], - cids: ref seq[Cid], - emptyBlock: seq[byte], -): Future[?!(Natural, Natural)] {.async.} = - ## Prepare data for decoding - ## `encoded` - the encoded manifest - ## `step` - the current step - ## `data` - the data to be prepared - ## `parityData` - the parityData to be prepared - ## `cids` - cids of prepared data - ## `emptyBlock` - the empty block to be used for padding - ## - - let - strategy = encoded.protectedStrategy.init( - firstIndex = 0, lastIndex = encoded.blocksCount - 1, iterations = encoded.steps - ) - indices = toSeq(strategy.getIndices(step)) - pendingBlocksIter = self.getPendingBlocks(encoded, indices) - - var - dataPieces = 0 - parityPieces = 0 - resolved = 0 - for fut in pendingBlocksIter: - # Continue to receive blocks until we have just enough for decoding - # or no more blocks can arrive - if resolved >= encoded.ecK: - break - - let (blkOrErr, idx) = await fut - without blk =? blkOrErr, err: - trace "Failed retrieving a block", idx, treeCid = encoded.treeCid, msg = err.msg - continue - - let pos = indexToPos(encoded.steps, idx, step) - - logScope: - cid = blk.cid - idx = idx - pos = pos - step = step - empty = blk.isEmpty - - cids[idx] = blk.cid - if idx >= encoded.rounded: - trace "Retrieved parity block" - shallowCopy( - parityData[pos - encoded.ecK], if blk.isEmpty: emptyBlock else: blk.data - ) - parityPieces.inc - else: - trace "Retrieved data block" - shallowCopy(data[pos], if blk.isEmpty: emptyBlock else: blk.data) - dataPieces.inc - - resolved.inc - - return success (dataPieces.Natural, parityPieces.Natural) - -proc init*( - _: type EncodingParams, - manifest: Manifest, - ecK: Natural, - ecM: Natural, - strategy: StrategyType, -): ?!EncodingParams = - if ecK > manifest.blocksCount: - let exc = (ref InsufficientBlocksError)( - msg: - "Unable to encode manifest, not enough blocks, ecK = " & $ecK & - ", blocksCount = " & $manifest.blocksCount, - minSize: ecK.NBytes * manifest.blockSize, - ) - return failure(exc) - - let - rounded = roundUp(manifest.blocksCount, ecK) - steps = divUp(rounded, ecK) - blocksCount = rounded + (steps * ecM) - - success EncodingParams( - ecK: ecK, - ecM: ecM, - rounded: rounded, - steps: steps, - blocksCount: blocksCount, - strategy: strategy, - ) - -proc leopardEncodeTask(tp: Taskpool, task: ptr EncodeTask) {.gcsafe.} = - # Task suitable for running in taskpools - look, no GC! - let encoder = - task[].erasure.encoderProvider(task[].blockSize, task[].blocksLen, task[].parityLen) - defer: - encoder.release() - discard task[].signal.fireSync() - - if ( - let res = - encoder.encode(task[].blocks, task[].parity, task[].blocksLen, task[].parityLen) - res.isErr - ): - warn "Error from leopard encoder backend!", error = $res.error - - task[].success.store(false) - else: - task[].success.store(true) - -proc asyncEncode*( - self: Erasure, - blockSize, blocksLen, parityLen: int, - blocks: ref seq[seq[byte]], - parity: ptr UncheckedArray[ptr UncheckedArray[byte]], -): Future[?!void] {.async: (raises: [CancelledError]).} = - without threadPtr =? ThreadSignalPtr.new(): - return failure("Unable to create thread signal") - - defer: - threadPtr.close().expect("closing once works") - - var data = makeUncheckedArray(blocks) - - defer: - dealloc(data) - - ## Create an ecode task with block data - var task = EncodeTask( - erasure: addr self, - blockSize: blockSize, - blocksLen: blocksLen, - parityLen: parityLen, - blocks: data, - parity: parity, - signal: threadPtr, - ) - - doAssert self.taskPool.numThreads > 1, - "Must have at least one separate thread or signal will never be fired" - self.taskPool.spawn leopardEncodeTask(self.taskPool, addr task) - let threadFut = threadPtr.wait() - - if joinErr =? catch(await threadFut.join()).errorOption: - if err =? catch(await noCancel threadFut).errorOption: - return failure(err) - if joinErr of CancelledError: - raise (ref CancelledError) joinErr - else: - return failure(joinErr) - - if not task.success.load(): - return failure("Leopard encoding task failed") - - success() - -proc encodeData( - self: Erasure, manifest: Manifest, params: EncodingParams -): Future[?!Manifest] {.async.} = - ## Encode blocks pointed to by the protected manifest - ## - ## `manifest` - the manifest to encode - ## - logScope: - steps = params.steps - rounded_blocks = params.rounded - blocks_count = params.blocksCount - ecK = params.ecK - ecM = params.ecM - - var - cids = seq[Cid].new() - emptyBlock = newSeq[byte](manifest.blockSize.int) - - cids[].setLen(params.blocksCount) - - try: - for step in 0 ..< params.steps: - # TODO: Don't allocate a new seq every time, allocate once and zero out - var - data = seq[seq[byte]].new() # number of blocks to encode - parity = createDoubleArray(params.ecM, manifest.blockSize.int) - defer: - freeDoubleArray(parity, params.ecM) - - data[].setLen(params.ecK) - # TODO: this is a tight blocking loop so we sleep here to allow - # other events to be processed, this should be addressed - # by threading - await sleepAsync(10.millis) - - without resolved =? - (await self.prepareEncodingData(manifest, params, step, data, cids, emptyBlock)), - err: - trace "Unable to prepare data", error = err.msg - return failure(err) - - trace "Erasure coding data", data = data[].len - - try: - if err =? ( - await self.asyncEncode( - manifest.blockSize.int, params.ecK, params.ecM, data, parity - ) - ).errorOption: - return failure(err) - except CancelledError as exc: - raise exc - - var idx = params.rounded + step - for j in 0 ..< params.ecM: - var innerPtr: ptr UncheckedArray[byte] = parity[][j] - without blk =? bt.Block.new(innerPtr.toOpenArray(0, manifest.blockSize.int - 1)), - error: - trace "Unable to create parity block", err = error.msg - return failure(error) - - trace "Adding parity block", cid = blk.cid, idx - cids[idx] = blk.cid - if error =? (await self.store.putBlock(blk)).errorOption: - warn "Unable to store block!", cid = blk.cid, msg = error.msg - return failure("Unable to store block!") - idx.inc(params.steps) - - without tree =? CodexTree.init(cids[]), err: - return failure(err) - - without treeCid =? tree.rootCid, err: - return failure(err) - - if err =? (await self.store.putAllProofs(tree)).errorOption: - return failure(err) - - let encodedManifest = Manifest.new( - manifest = manifest, - treeCid = treeCid, - datasetSize = (manifest.blockSize.int * params.blocksCount).NBytes, - ecK = params.ecK, - ecM = params.ecM, - strategy = params.strategy, - ) - - trace "Encoded data successfully", treeCid, blocksCount = params.blocksCount - success encodedManifest - except CancelledError as exc: - trace "Erasure coding encoding cancelled" - raise exc # cancellation needs to be propagated - except CatchableError as exc: - trace "Erasure coding encoding error", exc = exc.msg - return failure(exc) - -proc encode*( - self: Erasure, - manifest: Manifest, - blocks: Natural, - parity: Natural, - strategy = SteppedStrategy, -): Future[?!Manifest] {.async.} = - ## Encode a manifest into one that is erasure protected. - ## - ## `manifest` - the original manifest to be encoded - ## `blocks` - the number of blocks to be encoded - K - ## `parity` - the number of parity blocks to generate - M - ## - - without params =? EncodingParams.init(manifest, blocks.int, parity.int, strategy), err: - return failure(err) - - without encodedManifest =? await self.encodeData(manifest, params), err: - return failure(err) - - return success encodedManifest - -proc leopardDecodeTask(tp: Taskpool, task: ptr DecodeTask) {.gcsafe.} = - # Task suitable for running in taskpools - look, no GC! - let decoder = - task[].erasure.decoderProvider(task[].blockSize, task[].blocksLen, task[].parityLen) - defer: - decoder.release() - discard task[].signal.fireSync() - - if ( - let res = decoder.decode( - task[].blocks, - task[].parity, - task[].recovered, - task[].blocksLen, - task[].parityLen, - task[].recoveredLen, - ) - res.isErr - ): - warn "Error from leopard decoder backend!", error = $res.error - task[].success.store(false) - else: - task[].success.store(true) - -proc asyncDecode*( - self: Erasure, - blockSize, blocksLen, parityLen: int, - blocks, parity: ref seq[seq[byte]], - recovered: ptr UncheckedArray[ptr UncheckedArray[byte]], -): Future[?!void] {.async: (raises: [CancelledError]).} = - without threadPtr =? ThreadSignalPtr.new(): - return failure("Unable to create thread signal") - - defer: - threadPtr.close().expect("closing once works") - - var - blockData = makeUncheckedArray(blocks) - parityData = makeUncheckedArray(parity) - - defer: - dealloc(blockData) - dealloc(parityData) - - ## Create an decode task with block data - var task = DecodeTask( - erasure: addr self, - blockSize: blockSize, - blocksLen: blocksLen, - parityLen: parityLen, - recoveredLen: blocksLen, - blocks: blockData, - parity: parityData, - recovered: recovered, - signal: threadPtr, - ) - - doAssert self.taskPool.numThreads > 1, - "Must have at least one separate thread or signal will never be fired" - self.taskPool.spawn leopardDecodeTask(self.taskPool, addr task) - let threadFut = threadPtr.wait() - - if joinErr =? catch(await threadFut.join()).errorOption: - if err =? catch(await noCancel threadFut).errorOption: - return failure(err) - if joinErr of CancelledError: - raise (ref CancelledError) joinErr - else: - return failure(joinErr) - - if not task.success.load(): - return failure("Leopard decoding task failed") - - success() - -proc decodeInternal( - self: Erasure, encoded: Manifest -): Future[?!(ref seq[Cid], seq[Natural])] {.async.} = - logScope: - steps = encoded.steps - rounded_blocks = encoded.rounded - new_manifest = encoded.blocksCount - - var - cids = seq[Cid].new() - recoveredIndices = newSeq[Natural]() - decoder = self.decoderProvider(encoded.blockSize.int, encoded.ecK, encoded.ecM) - emptyBlock = newSeq[byte](encoded.blockSize.int) - - cids[].setLen(encoded.blocksCount) - try: - for step in 0 ..< encoded.steps: - # TODO: this is a tight blocking loop so we sleep here to allow - # other events to be processed, this should be addressed - # by threading - await sleepAsync(10.millis) - - var - data = seq[seq[byte]].new() - parityData = seq[seq[byte]].new() - recovered = createDoubleArray(encoded.ecK, encoded.blockSize.int) - defer: - freeDoubleArray(recovered, encoded.ecK) - - data[].setLen(encoded.ecK) # set len to K - parityData[].setLen(encoded.ecM) # set len to M - - without (dataPieces, _) =? ( - await self.prepareDecodingData( - encoded, step, data, parityData, cids, emptyBlock - ) - ), err: - trace "Unable to prepare data", error = err.msg - return failure(err) - - if dataPieces >= encoded.ecK: - trace "Retrieved all the required data blocks" - continue - - trace "Erasure decoding data" - try: - if err =? ( - await self.asyncDecode( - encoded.blockSize.int, encoded.ecK, encoded.ecM, data, parityData, recovered - ) - ).errorOption: - return failure(err) - except CancelledError as exc: - raise exc - - for i in 0 ..< encoded.ecK: - let idx = i * encoded.steps + step - if data[i].len <= 0 and not cids[idx].isEmpty: - var innerPtr: ptr UncheckedArray[byte] = recovered[][i] - - without blk =? bt.Block.new( - innerPtr.toOpenArray(0, encoded.blockSize.int - 1) - ), error: - trace "Unable to create block!", exc = error.msg - return failure(error) - - trace "Recovered block", cid = blk.cid, index = i - if error =? (await self.store.putBlock(blk)).errorOption: - warn "Unable to store block!", cid = blk.cid, msg = error.msg - return failure("Unable to store block!") - - self.store.completeBlock(BlockAddress.init(encoded.treeCid, idx), blk) - - cids[idx] = blk.cid - recoveredIndices.add(idx) - except CancelledError as exc: - trace "Erasure coding decoding cancelled" - raise exc # cancellation needs to be propagated - except CatchableError as exc: - trace "Erasure coding decoding error", exc = exc.msg - return failure(exc) - finally: - decoder.release() - - return (cids, recoveredIndices).success - -proc decode*(self: Erasure, encoded: Manifest): Future[?!Manifest] {.async.} = - ## Decode a protected manifest into it's original - ## manifest - ## - ## `encoded` - the encoded (protected) manifest to - ## be recovered - ## - - without (cids, recoveredIndices) =? (await self.decodeInternal(encoded)), err: - return failure(err) - - without tree =? CodexTree.init(cids[0 ..< encoded.originalBlocksCount]), err: - return failure(err) - - without treeCid =? tree.rootCid, err: - return failure(err) - - if treeCid != encoded.originalTreeCid: - return failure( - "Original tree root differs from the tree root computed out of recovered data" - ) - - let idxIter = - Iter[Natural].new(recoveredIndices).filter((i: Natural) => i < tree.leavesCount) - - if err =? (await self.store.putSomeProofs(tree, idxIter)).errorOption: - return failure(err) - - let decoded = Manifest.new(encoded) - - return decoded.success - -proc repair*(self: Erasure, encoded: Manifest): Future[?!void] {.async.} = - ## Repair a protected manifest by reconstructing the full dataset - ## - ## `encoded` - the encoded (protected) manifest to - ## be repaired - ## - - without (cids, _) =? (await self.decodeInternal(encoded)), err: - return failure(err) - - without tree =? CodexTree.init(cids[0 ..< encoded.originalBlocksCount]), err: - return failure(err) - - without treeCid =? tree.rootCid, err: - return failure(err) - - if treeCid != encoded.originalTreeCid: - return failure( - "Original tree root differs from the tree root computed out of recovered data" - ) - - if err =? (await self.store.putAllProofs(tree)).errorOption: - return failure(err) - - without repaired =? ( - await self.encode( - Manifest.new(encoded), encoded.ecK, encoded.ecM, encoded.protectedStrategy - ) - ), err: - return failure(err) - - if repaired.treeCid != encoded.treeCid: - return failure( - "Original tree root differs from the repaired tree root encoded out of recovered data" - ) - - return success() - -proc start*(self: Erasure) {.async.} = - return - -proc stop*(self: Erasure) {.async.} = - return - -proc new*( - T: type Erasure, - store: BlockStore, - encoderProvider: EncoderProvider, - decoderProvider: DecoderProvider, - taskPool: Taskpool, -): Erasure = - ## Create a new Erasure instance for encoding and decoding manifests - ## - Erasure( - store: store, - encoderProvider: encoderProvider, - decoderProvider: decoderProvider, - taskPool: taskPool, - ) diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index b899bb09..f7fb1cf3 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -32,25 +32,12 @@ proc encode*(manifest: Manifest): ?!seq[byte] = ## multicodec container (Dag-pb) for now ## - ?manifest.verify() var pbNode = initProtoBuffer() # NOTE: The `Data` field in the the `dag-pb` # contains the following protobuf `Message` # # ```protobuf - # Message VerificationInfo { - # bytes verifyRoot = 1; # Decimal encoded field-element - # repeated bytes slotRoots = 2; # Decimal encoded field-elements - # } - # Message ErasureInfo { - # optional uint32 ecK = 1; # number of encoded blocks - # optional uint32 ecM = 2; # number of parity blocks - # optional bytes originalTreeCid = 3; # cid of the original dataset - # optional uint32 originalDatasetSize = 4; # size of the original dataset - # optional VerificationInformation verification = 5; # verification information - # } - # # Message Header { # optional bytes treeCid = 1; # cid (root) of the tree # optional uint32 blockSize = 2; # size of a single block @@ -58,9 +45,8 @@ proc encode*(manifest: Manifest): ?!seq[byte] = # optional codec: MultiCodec = 4; # Dataset codec # optional hcodec: MultiCodec = 5 # Multihash codec # optional version: CidVersion = 6; # Cid version - # optional ErasureInfo erasure = 7; # erasure coding info - # optional filename: ?string = 8; # original filename - # optional mimetype: ?string = 9; # original mimetype + # optional filename: ?string = 7; # original filename + # optional mimetype: ?string = 8; # original mimetype # } # ``` # @@ -73,31 +59,11 @@ proc encode*(manifest: Manifest): ?!seq[byte] = header.write(5, manifest.hcodec.uint32) header.write(6, manifest.version.uint32) - if manifest.protected: - var erasureInfo = initProtoBuffer() - erasureInfo.write(1, manifest.ecK.uint32) - erasureInfo.write(2, manifest.ecM.uint32) - erasureInfo.write(3, manifest.originalTreeCid.data.buffer) - erasureInfo.write(4, manifest.originalDatasetSize.uint64) - erasureInfo.write(5, manifest.protectedStrategy.uint32) - - if manifest.verifiable: - var verificationInfo = initProtoBuffer() - verificationInfo.write(1, manifest.verifyRoot.data.buffer) - for slotRoot in manifest.slotRoots: - verificationInfo.write(2, slotRoot.data.buffer) - verificationInfo.write(3, manifest.cellSize.uint32) - verificationInfo.write(4, manifest.verifiableStrategy.uint32) - erasureInfo.write(6, verificationInfo) - - erasureInfo.finish() - header.write(7, erasureInfo) - if manifest.filename.isSome: - header.write(8, manifest.filename.get()) + header.write(7, manifest.filename.get()) if manifest.mimetype.isSome: - header.write(9, manifest.mimetype.get()) + header.write(8, manifest.mimetype.get()) pbNode.write(1, header) # set the treeCid as the data field pbNode.finish() @@ -111,22 +77,12 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = var pbNode = initProtoBuffer(data) pbHeader: ProtoBuffer - pbErasureInfo: ProtoBuffer - pbVerificationInfo: ProtoBuffer treeCidBuf: seq[byte] - originalTreeCid: seq[byte] datasetSize: uint64 codec: uint32 hcodec: uint32 version: uint32 blockSize: uint32 - originalDatasetSize: uint64 - ecK, ecM: uint32 - protectedStrategy: uint32 - verifyRoot: seq[byte] - slotRoots: seq[seq[byte]] - cellSize: uint32 - verifiableStrategy: uint32 filename: string mimetype: string @@ -153,98 +109,27 @@ proc decode*(_: type Manifest, data: openArray[byte]): ?!Manifest = if pbHeader.getField(6, version).isErr: return failure("Unable to decode `version` from manifest!") - if pbHeader.getField(7, pbErasureInfo).isErr: - return failure("Unable to decode `erasureInfo` from manifest!") - - if pbHeader.getField(8, filename).isErr: + if pbHeader.getField(7, filename).isErr: return failure("Unable to decode `filename` from manifest!") - if pbHeader.getField(9, mimetype).isErr: + if pbHeader.getField(8, mimetype).isErr: return failure("Unable to decode `mimetype` from manifest!") - let protected = pbErasureInfo.buffer.len > 0 - var verifiable = false - if protected: - if pbErasureInfo.getField(1, ecK).isErr: - return failure("Unable to decode `K` from manifest!") - - if pbErasureInfo.getField(2, ecM).isErr: - return failure("Unable to decode `M` from manifest!") - - if pbErasureInfo.getField(3, originalTreeCid).isErr: - return failure("Unable to decode `originalTreeCid` from manifest!") - - if pbErasureInfo.getField(4, originalDatasetSize).isErr: - return failure("Unable to decode `originalDatasetSize` from manifest!") - - if pbErasureInfo.getField(5, protectedStrategy).isErr: - return failure("Unable to decode `protectedStrategy` from manifest!") - - if pbErasureInfo.getField(6, pbVerificationInfo).isErr: - return failure("Unable to decode `verificationInfo` from manifest!") - - verifiable = pbVerificationInfo.buffer.len > 0 - if verifiable: - if pbVerificationInfo.getField(1, verifyRoot).isErr: - return failure("Unable to decode `verifyRoot` from manifest!") - - if pbVerificationInfo.getRequiredRepeatedField(2, slotRoots).isErr: - return failure("Unable to decode `slotRoots` from manifest!") - - if pbVerificationInfo.getField(3, cellSize).isErr: - return failure("Unable to decode `cellSize` from manifest!") - - if pbVerificationInfo.getField(4, verifiableStrategy).isErr: - return failure("Unable to decode `verifiableStrategy` from manifest!") - let treeCid = ?Cid.init(treeCidBuf).mapFailure var filenameOption = if filename.len == 0: string.none else: filename.some var mimetypeOption = if mimetype.len == 0: string.none else: mimetype.some - let self = - if protected: - Manifest.new( - treeCid = treeCid, - datasetSize = datasetSize.NBytes, - blockSize = blockSize.NBytes, - version = CidVersion(version), - hcodec = hcodec.MultiCodec, - codec = codec.MultiCodec, - ecK = ecK.int, - ecM = ecM.int, - originalTreeCid = ?Cid.init(originalTreeCid).mapFailure, - originalDatasetSize = originalDatasetSize.NBytes, - strategy = StrategyType(protectedStrategy), - filename = filenameOption, - mimetype = mimetypeOption, - ) - else: - Manifest.new( - treeCid = treeCid, - datasetSize = datasetSize.NBytes, - blockSize = blockSize.NBytes, - version = CidVersion(version), - hcodec = hcodec.MultiCodec, - codec = codec.MultiCodec, - filename = filenameOption, - mimetype = mimetypeOption, - ) - - ?self.verify() - - if verifiable: - let - verifyRootCid = ?Cid.init(verifyRoot).mapFailure - slotRootCids = slotRoots.mapIt(?Cid.init(it).mapFailure) - - return Manifest.new( - manifest = self, - verifyRoot = verifyRootCid, - slotRoots = slotRootCids, - cellSize = cellSize.NBytes, - strategy = StrategyType(verifiableStrategy), - ) + let self = Manifest.new( + treeCid = treeCid, + datasetSize = datasetSize.NBytes, + blockSize = blockSize.NBytes, + version = CidVersion(version), + hcodec = hcodec.MultiCodec, + codec = codec.MultiCodec, + filename = filenameOption, + mimetype = mimetypeOption, + ) self.success diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index fadc8c88..0823f804 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -35,24 +35,6 @@ type Manifest* = ref object of RootObj version: CidVersion # Cid version filename {.serialize.}: ?string # The filename of the content uploaded (optional) mimetype {.serialize.}: ?string # The mimetype of the content uploaded (optional) - case protected {.serialize.}: bool # Protected datasets have erasure coded info - of true: - ecK: int # Number of blocks to encode - ecM: int # Number of resulting parity blocks - originalTreeCid: Cid # The original root of the dataset being erasure coded - originalDatasetSize: NBytes - protectedStrategy: StrategyType # Indexing strategy used to build the slot roots - case verifiable {.serialize.}: bool - # Verifiable datasets can be used to generate storage proofs - of true: - verifyRoot: Cid # Root of the top level merkle tree built from slot roots - slotRoots: seq[Cid] # Individual slot root built from the original dataset blocks - cellSize: NBytes # Size of each slot cell - verifiableStrategy: StrategyType # Indexing strategy used to build the slot roots - else: - discard - else: - discard ############################################################ # Accessors @@ -73,54 +55,12 @@ func hcodec*(self: Manifest): MultiCodec = func codec*(self: Manifest): MultiCodec = self.codec -func protected*(self: Manifest): bool = - self.protected - -func ecK*(self: Manifest): int = - self.ecK - -func ecM*(self: Manifest): int = - self.ecM - -func originalTreeCid*(self: Manifest): Cid = - self.originalTreeCid - -func originalBlocksCount*(self: Manifest): int = - divUp(self.originalDatasetSize.int, self.blockSize.int) - -func originalDatasetSize*(self: Manifest): NBytes = - self.originalDatasetSize - func treeCid*(self: Manifest): Cid = self.treeCid func blocksCount*(self: Manifest): int = divUp(self.datasetSize.int, self.blockSize.int) -func verifiable*(self: Manifest): bool = - bool (self.protected and self.verifiable) - -func verifyRoot*(self: Manifest): Cid = - self.verifyRoot - -func slotRoots*(self: Manifest): seq[Cid] = - self.slotRoots - -func numSlots*(self: Manifest): int = - self.ecK + self.ecM - -func cellSize*(self: Manifest): NBytes = - self.cellSize - -func protectedStrategy*(self: Manifest): StrategyType = - self.protectedStrategy - -func verifiableStrategy*(self: Manifest): StrategyType = - self.verifiableStrategy - -func numSlotBlocks*(self: Manifest): int = - divUp(self.blocksCount, self.numSlots) - func filename*(self: Manifest): ?string = self.filename @@ -141,51 +81,17 @@ func isManifest*(mc: MultiCodec): ?!bool = # Various sizes and verification ############################################################ -func rounded*(self: Manifest): int = - ## Number of data blocks in *protected* manifest including padding at the end - roundUp(self.originalBlocksCount, self.ecK) - -func steps*(self: Manifest): int = - ## Number of EC groups in *protected* manifest - divUp(self.rounded, self.ecK) - -func verify*(self: Manifest): ?!void = - ## Check manifest correctness - ## - - if self.protected and (self.blocksCount != self.steps * (self.ecK + self.ecM)): - return - failure newException(CodexError, "Broken manifest: wrong originalBlocksCount") - - return success() - func `==`*(a, b: Manifest): bool = (a.treeCid == b.treeCid) and (a.datasetSize == b.datasetSize) and (a.blockSize == b.blockSize) and (a.version == b.version) and (a.hcodec == b.hcodec) and - (a.codec == b.codec) and (a.protected == b.protected) and (a.filename == b.filename) and - (a.mimetype == b.mimetype) and ( - if a.protected: - (a.ecK == b.ecK) and (a.ecM == b.ecM) and (a.originalTreeCid == b.originalTreeCid) and - (a.originalDatasetSize == b.originalDatasetSize) and - (a.protectedStrategy == b.protectedStrategy) and (a.verifiable == b.verifiable) and - ( - if a.verifiable: - (a.verifyRoot == b.verifyRoot) and (a.slotRoots == b.slotRoots) and - (a.cellSize == b.cellSize) and ( - a.verifiableStrategy == b.verifiableStrategy - ) - else: - true - ) - else: - true - ) + (a.codec == b.codec) and (a.filename == b.filename) and + (a.mimetype == b.mimetype) func `$`*(self: Manifest): string = result = "treeCid: " & $self.treeCid & ", datasetSize: " & $self.datasetSize & ", blockSize: " & $self.blockSize & ", version: " & $self.version & ", hcodec: " & $self.hcodec & - ", codec: " & $self.codec & ", protected: " & $self.protected + ", codec: " & $self.codec if self.filename.isSome: result &= ", filename: " & $self.filename @@ -193,20 +99,6 @@ func `$`*(self: Manifest): string = if self.mimetype.isSome: result &= ", mimetype: " & $self.mimetype - result &= ( - if self.protected: - ", ecK: " & $self.ecK & ", ecM: " & $self.ecM & ", originalTreeCid: " & - $self.originalTreeCid & ", originalDatasetSize: " & $self.originalDatasetSize & - ", verifiable: " & $self.verifiable & ( - if self.verifiable: - ", verifyRoot: " & $self.verifyRoot & ", slotRoots: " & $self.slotRoots - else: - "" - ) - else: - "" - ) - return result ############################################################ @@ -221,7 +113,6 @@ func new*( version: CidVersion = CIDv1, hcodec = Sha256HashCodec, codec = BlockCodec, - protected = false, filename: ?string = string.none, mimetype: ?string = string.none, ): Manifest = @@ -232,132 +123,10 @@ func new*( version: version, codec: codec, hcodec: hcodec, - protected: protected, filename: filename, mimetype: mimetype, ) -func new*( - T: type Manifest, - manifest: Manifest, - treeCid: Cid, - datasetSize: NBytes, - ecK, ecM: int, - strategy = SteppedStrategy, -): Manifest = - ## Create an erasure protected dataset from an - ## unprotected one - ## - - Manifest( - treeCid: treeCid, - datasetSize: datasetSize, - version: manifest.version, - codec: manifest.codec, - hcodec: manifest.hcodec, - blockSize: manifest.blockSize, - protected: true, - ecK: ecK, - ecM: ecM, - originalTreeCid: manifest.treeCid, - originalDatasetSize: manifest.datasetSize, - protectedStrategy: strategy, - filename: manifest.filename, - mimetype: manifest.mimetype, - ) - -func new*(T: type Manifest, manifest: Manifest): Manifest = - ## Create an unprotected dataset from an - ## erasure protected one - ## - - Manifest( - treeCid: manifest.originalTreeCid, - datasetSize: manifest.originalDatasetSize, - version: manifest.version, - codec: manifest.codec, - hcodec: manifest.hcodec, - blockSize: manifest.blockSize, - protected: false, - filename: manifest.filename, - mimetype: manifest.mimetype, - ) - -func new*( - T: type Manifest, - treeCid: Cid, - datasetSize: NBytes, - blockSize: NBytes, - version: CidVersion, - hcodec: MultiCodec, - codec: MultiCodec, - ecK: int, - ecM: int, - originalTreeCid: Cid, - originalDatasetSize: NBytes, - strategy = SteppedStrategy, - filename: ?string = string.none, - mimetype: ?string = string.none, -): Manifest = - Manifest( - treeCid: treeCid, - datasetSize: datasetSize, - blockSize: blockSize, - version: version, - hcodec: hcodec, - codec: codec, - protected: true, - ecK: ecK, - ecM: ecM, - originalTreeCid: originalTreeCid, - originalDatasetSize: originalDatasetSize, - protectedStrategy: strategy, - filename: filename, - mimetype: mimetype, - ) - -func new*( - T: type Manifest, - manifest: Manifest, - verifyRoot: Cid, - slotRoots: openArray[Cid], - cellSize = DefaultCellSize, - strategy = LinearStrategy, -): ?!Manifest = - ## Create a verifiable dataset from an - ## protected one - ## - - if not manifest.protected: - return failure newException( - CodexError, "Can create verifiable manifest only from protected manifest." - ) - - if slotRoots.len != manifest.numSlots: - return failure newException(CodexError, "Wrong number of slot roots.") - - success Manifest( - treeCid: manifest.treeCid, - datasetSize: manifest.datasetSize, - version: manifest.version, - codec: manifest.codec, - hcodec: manifest.hcodec, - blockSize: manifest.blockSize, - protected: true, - ecK: manifest.ecK, - ecM: manifest.ecM, - originalTreeCid: manifest.originalTreeCid, - originalDatasetSize: manifest.originalDatasetSize, - protectedStrategy: manifest.protectedStrategy, - verifiable: true, - verifyRoot: verifyRoot, - slotRoots: @slotRoots, - cellSize: cellSize, - verifiableStrategy: strategy, - filename: manifest.filename, - mimetype: manifest.mimetype, - ) - func new*(T: type Manifest, data: openArray[byte]): ?!Manifest = ## Create a manifest instance from given data ## diff --git a/codex/market.nim b/codex/market.nim deleted file mode 100644 index 968f204e..00000000 --- a/codex/market.nim +++ /dev/null @@ -1,313 +0,0 @@ -import pkg/chronos -import pkg/questionable -import pkg/ethers/erc20 -import ./contracts/requests -import ./contracts/proofs -import ./clock -import ./errors -import ./periods - -export chronos -export questionable -export requests -export proofs -export SecondsSince1970 -export periods - -type - Market* = ref object of RootObj - MarketError* = object of CodexError - SlotStateMismatchError* = object of MarketError - SlotReservationNotAllowedError* = object of MarketError - ProofInvalidError* = object of MarketError - Subscription* = ref object of RootObj - OnRequest* = - proc(id: RequestId, ask: StorageAsk, expiry: uint64) {.gcsafe, raises: [].} - OnFulfillment* = proc(requestId: RequestId) {.gcsafe, raises: [].} - OnSlotFilled* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, raises: [].} - OnSlotFreed* = proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, raises: [].} - OnSlotReservationsFull* = - proc(requestId: RequestId, slotIndex: uint64) {.gcsafe, raises: [].} - OnRequestCancelled* = proc(requestId: RequestId) {.gcsafe, raises: [].} - OnRequestFailed* = proc(requestId: RequestId) {.gcsafe, raises: [].} - OnProofSubmitted* = proc(id: SlotId) {.gcsafe, raises: [].} - ProofChallenge* = array[32, byte] - - # Marketplace events -- located here due to the Market abstraction - MarketplaceEvent* = Event - StorageRequested* = object of MarketplaceEvent - requestId*: RequestId - ask*: StorageAsk - expiry*: uint64 - - SlotFilled* = object of MarketplaceEvent - requestId* {.indexed.}: RequestId - slotIndex*: uint64 - - SlotFreed* = object of MarketplaceEvent - requestId* {.indexed.}: RequestId - slotIndex*: uint64 - - SlotReservationsFull* = object of MarketplaceEvent - requestId* {.indexed.}: RequestId - slotIndex*: uint64 - - RequestFulfilled* = object of MarketplaceEvent - requestId* {.indexed.}: RequestId - - RequestCancelled* = object of MarketplaceEvent - requestId* {.indexed.}: RequestId - - RequestFailed* = object of MarketplaceEvent - requestId* {.indexed.}: RequestId - - ProofSubmitted* = object of MarketplaceEvent - id*: SlotId - -method loadConfig*( - market: Market -): Future[?!void] {.base, async: (raises: [CancelledError]).} = - raiseAssert("not implemented") - -method getZkeyHash*( - market: Market -): Future[?string] {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method getSigner*( - market: Market -): Future[Address] {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method periodicity*( - market: Market -): Future[Periodicity] {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method proofTimeout*( - market: Market -): Future[uint64] {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method repairRewardPercentage*( - market: Market -): Future[uint8] {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method requestDurationLimit*(market: Market): Future[uint64] {.base, async.} = - raiseAssert("not implemented") - -method proofDowntime*( - market: Market -): Future[uint8] {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method getPointer*(market: Market, slotId: SlotId): Future[uint8] {.base, async.} = - raiseAssert("not implemented") - -proc inDowntime*(market: Market, slotId: SlotId): Future[bool] {.async.} = - let downtime = await market.proofDowntime - let pntr = await market.getPointer(slotId) - return pntr < downtime - -method requestStorage*( - market: Market, request: StorageRequest -) {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method myRequests*(market: Market): Future[seq[RequestId]] {.base, async.} = - raiseAssert("not implemented") - -method mySlots*(market: Market): Future[seq[SlotId]] {.base, async.} = - raiseAssert("not implemented") - -method getRequest*( - market: Market, id: RequestId -): Future[?StorageRequest] {.base, async: (raises: [CancelledError]).} = - raiseAssert("not implemented") - -method requestState*( - market: Market, requestId: RequestId -): Future[?RequestState] {.base, async.} = - raiseAssert("not implemented") - -method slotState*( - market: Market, slotId: SlotId -): Future[SlotState] {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method getRequestEnd*( - market: Market, id: RequestId -): Future[SecondsSince1970] {.base, async.} = - raiseAssert("not implemented") - -method requestExpiresAt*( - market: Market, id: RequestId -): Future[SecondsSince1970] {.base, async.} = - raiseAssert("not implemented") - -method getHost*( - market: Market, requestId: RequestId, slotIndex: uint64 -): Future[?Address] {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method currentCollateral*( - market: Market, slotId: SlotId -): Future[UInt256] {.base, async: (raises: [MarketError, CancelledError]).} = - raiseAssert("not implemented") - -method getActiveSlot*(market: Market, slotId: SlotId): Future[?Slot] {.base, async.} = - raiseAssert("not implemented") - -method fillSlot*( - market: Market, - requestId: RequestId, - slotIndex: uint64, - proof: Groth16Proof, - collateral: UInt256, -) {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method freeSlot*( - market: Market, slotId: SlotId -) {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method withdrawFunds*( - market: Market, requestId: RequestId -) {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method subscribeRequests*( - market: Market, callback: OnRequest -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method isProofRequired*(market: Market, id: SlotId): Future[bool] {.base, async.} = - raiseAssert("not implemented") - -method willProofBeRequired*(market: Market, id: SlotId): Future[bool] {.base, async.} = - raiseAssert("not implemented") - -method getChallenge*( - market: Market, id: SlotId -): Future[ProofChallenge] {.base, async.} = - raiseAssert("not implemented") - -method submitProof*( - market: Market, id: SlotId, proof: Groth16Proof -) {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method markProofAsMissing*( - market: Market, id: SlotId, period: Period -) {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method canMarkProofAsMissing*( - market: Market, id: SlotId, period: Period -): Future[bool] {.base, async: (raises: [CancelledError]).} = - raiseAssert("not implemented") - -method reserveSlot*( - market: Market, requestId: RequestId, slotIndex: uint64 -) {.base, async: (raises: [CancelledError, MarketError]).} = - raiseAssert("not implemented") - -method canReserveSlot*( - market: Market, requestId: RequestId, slotIndex: uint64 -): Future[bool] {.base, async.} = - raiseAssert("not implemented") - -method subscribeFulfillment*( - market: Market, callback: OnFulfillment -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeFulfillment*( - market: Market, requestId: RequestId, callback: OnFulfillment -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeSlotFilled*( - market: Market, callback: OnSlotFilled -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeSlotFilled*( - market: Market, requestId: RequestId, slotIndex: uint64, callback: OnSlotFilled -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeSlotFreed*( - market: Market, callback: OnSlotFreed -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeSlotReservationsFull*( - market: Market, callback: OnSlotReservationsFull -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeRequestCancelled*( - market: Market, callback: OnRequestCancelled -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeRequestCancelled*( - market: Market, requestId: RequestId, callback: OnRequestCancelled -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeRequestFailed*( - market: Market, callback: OnRequestFailed -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeRequestFailed*( - market: Market, requestId: RequestId, callback: OnRequestFailed -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method subscribeProofSubmission*( - market: Market, callback: OnProofSubmitted -): Future[Subscription] {.base, async.} = - raiseAssert("not implemented") - -method unsubscribe*(subscription: Subscription) {.base, async.} = - raiseAssert("not implemented") - -method queryPastSlotFilledEvents*( - market: Market, fromBlock: BlockTag -): Future[seq[SlotFilled]] {.base, async.} = - raiseAssert("not implemented") - -method queryPastSlotFilledEvents*( - market: Market, blocksAgo: int -): Future[seq[SlotFilled]] {.base, async.} = - raiseAssert("not implemented") - -method queryPastSlotFilledEvents*( - market: Market, fromTime: SecondsSince1970 -): Future[seq[SlotFilled]] {.base, async.} = - raiseAssert("not implemented") - -method queryPastStorageRequestedEvents*( - market: Market, fromBlock: BlockTag -): Future[seq[StorageRequested]] {.base, async.} = - raiseAssert("not implemented") - -method queryPastStorageRequestedEvents*( - market: Market, blocksAgo: int -): Future[seq[StorageRequested]] {.base, async.} = - raiseAssert("not implemented") - -method slotCollateral*( - market: Market, requestId: RequestId, slotIndex: uint64 -): Future[?!UInt256] {.base, async: (raises: [CancelledError]).} = - raiseAssert("not implemented") - -method slotCollateral*( - market: Market, collateralPerSlot: UInt256, slotState: SlotState -): ?!UInt256 {.base, gcsafe, raises: [].} = - raiseAssert("not implemented") diff --git a/codex/node.nim b/codex/node.nim index e5c8926e..5b8525a5 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -19,7 +19,6 @@ import pkg/taskpools import pkg/questionable import pkg/questionable/results import pkg/chronos -import pkg/poseidon2 import pkg/libp2p/[switch, multicodec, multihash] import pkg/libp2p/stream/bufferstream @@ -29,7 +28,6 @@ import pkg/libp2p/routing_record import pkg/libp2p/signed_envelope import ./chunker -import ./slots import ./clock import ./blocktype as bt import ./manifest @@ -37,9 +35,7 @@ import ./merkletree import ./stores import ./blockexchange import ./streams -import ./erasure import ./discovery -import ./contracts import ./indexingstrategy import ./utils import ./errors @@ -58,23 +54,13 @@ const BatchRefillThreshold = 0.75 # Refill when 75% of window completes type - Contracts* = - tuple[ - client: ?ClientInteractions, - host: ?HostInteractions, - validator: ?ValidatorInteractions, - ] - CodexNode* = object switch: Switch networkId: PeerId networkStore: NetworkStore engine: BlockExcEngine - prover: ?Prover discovery: Discovery - contracts*: Contracts clock*: Clock - storage*: Contracts taskpool: Taskpool trackedFutures: TrackedFutures @@ -319,20 +305,6 @@ proc streamEntireDataset( var jobs: seq[Future[void]] let stream = LPStream(StoreStream.new(self.networkStore, manifest, pad = false)) - if manifest.protected: - # Retrieve, decode and save to the local store all EС groups - proc erasureJob(): Future[void] {.async: (raises: []).} = - try: - # Spawn an erasure decoding job - let erasure = Erasure.new( - self.networkStore, leoEncoderProvider, leoDecoderProvider, self.taskpool - ) - without _ =? (await erasure.decode(manifest)), error: - error "Unable to erasure decode manifest", manifestCid, exc = error.msg - except CatchableError as exc: - trace "Error erasure decoding manifest", manifestCid, exc = exc.msg - - jobs.add(erasureJob()) jobs.add(self.fetchDatasetAsync(manifest, fetchLocal = false)) @@ -530,295 +502,11 @@ proc iterateManifests*(self: CodexNodeRef, onManifest: OnManifest) {.async.} = onManifest(cid, manifest) -proc setupRequest( - self: CodexNodeRef, - cid: Cid, - duration: uint64, - proofProbability: UInt256, - nodes: uint, - tolerance: uint, - pricePerBytePerSecond: UInt256, - collateralPerByte: UInt256, - expiry: uint64, -): Future[?!StorageRequest] {.async.} = - ## Setup slots for a given dataset - ## - - let - ecK = nodes - tolerance - ecM = tolerance - - logScope: - cid = cid - duration = duration - nodes = nodes - tolerance = tolerance - pricePerBytePerSecond = pricePerBytePerSecond - proofProbability = proofProbability - collateralPerByte = collateralPerByte - expiry = expiry - ecK = ecK - ecM = ecM - - trace "Setting up slots" - - without manifest =? await self.fetchManifest(cid), error: - trace "Unable to fetch manifest for cid" - return failure error - - # Erasure code the dataset according to provided parameters - let erasure = Erasure.new( - self.networkStore.localStore, leoEncoderProvider, leoDecoderProvider, self.taskpool - ) - - without encoded =? (await erasure.encode(manifest, ecK, ecM)), error: - trace "Unable to erasure code dataset" - return failure(error) - - without builder =? Poseidon2Builder.new(self.networkStore.localStore, encoded), err: - trace "Unable to create slot builder" - return failure(err) - - without verifiable =? (await builder.buildManifest()), err: - trace "Unable to build verifiable manifest" - return failure(err) - - without manifestBlk =? await self.storeManifest(verifiable), err: - trace "Unable to store verifiable manifest" - return failure(err) - - let - verifyRoot = - if builder.verifyRoot.isNone: - return failure("No slots root") - else: - builder.verifyRoot.get.toBytes - - request = StorageRequest( - ask: StorageAsk( - slots: verifiable.numSlots.uint64, - slotSize: builder.slotBytes.uint64, - duration: duration, - proofProbability: proofProbability, - pricePerBytePerSecond: pricePerBytePerSecond, - collateralPerByte: collateralPerByte, - maxSlotLoss: tolerance, - ), - content: StorageContent(cid: manifestBlk.cid, merkleRoot: verifyRoot), - expiry: expiry, - ) - - trace "Request created", request = $request - success request - -proc requestStorage*( - self: CodexNodeRef, - cid: Cid, - duration: uint64, - proofProbability: UInt256, - nodes: uint, - tolerance: uint, - pricePerBytePerSecond: UInt256, - collateralPerByte: UInt256, - expiry: uint64, -): Future[?!PurchaseId] {.async.} = - ## Initiate a request for storage sequence, this might - ## be a multistep procedure. - ## - - logScope: - cid = cid - duration = duration - nodes = nodes - tolerance = tolerance - pricePerBytePerSecond = pricePerBytePerSecond - proofProbability = proofProbability - collateralPerByte = collateralPerByte - expiry = expiry - now = self.clock.now - - trace "Received a request for storage!" - - without contracts =? self.contracts.client: - trace "Purchasing not available" - return failure "Purchasing not available" - - without request =? ( - await self.setupRequest( - cid, duration, proofProbability, nodes, tolerance, pricePerBytePerSecond, - collateralPerByte, expiry, - ) - ), err: - trace "Unable to setup request" - return failure err - - let purchase = await contracts.purchasing.purchase(request) - success purchase.id - -proc onStore( - self: CodexNodeRef, - request: StorageRequest, - expiry: SecondsSince1970, - slotIdx: uint64, - blocksCb: BlocksCb, - isRepairing: bool = false, -): Future[?!void] {.async: (raises: [CancelledError]).} = - ## store data in local storage - ## - - let cid = request.content.cid - - logScope: - cid = $cid - slotIdx = slotIdx - - trace "Received a request to store a slot" - - without manifest =? (await self.fetchManifest(cid)), err: - trace "Unable to fetch manifest for cid", cid, err = err.msg - return failure(err) - - without builder =? - Poseidon2Builder.new(self.networkStore, manifest, manifest.verifiableStrategy), err: - trace "Unable to create slots builder", err = err.msg - return failure(err) - - if slotIdx > manifest.slotRoots.high.uint64: - trace "Slot index not in manifest", slotIdx - return failure(newException(CodexError, "Slot index not in manifest")) - - proc updateExpiry( - blocks: seq[bt.Block] - ): Future[?!void] {.async: (raises: [CancelledError]).} = - trace "Updating expiry for blocks", blocks = blocks.len - - let ensureExpiryFutures = - blocks.mapIt(self.networkStore.ensureExpiry(it.cid, expiry)) - - let res = await allFinishedFailed[?!void](ensureExpiryFutures) - if res.failure.len > 0: - trace "Some blocks failed to update expiry", len = res.failure.len - return failure("Some blocks failed to update expiry (" & $res.failure.len & " )") - - if not blocksCb.isNil and err =? (await blocksCb(blocks)).errorOption: - trace "Unable to process blocks", err = err.msg - return failure(err) - - return success() - - if slotIdx > int.high.uint64: - error "Cannot cast slot index to int", slotIndex = slotIdx - return - - if isRepairing: - trace "start repairing slot", slotIdx - try: - let erasure = Erasure.new( - self.networkStore, leoEncoderProvider, leoDecoderProvider, self.taskpool - ) - if err =? (await erasure.repair(manifest)).errorOption: - error "Unable to erasure decode repairing manifest", - cid = manifest.treeCid, exc = err.msg - return failure(err) - except CatchableError as exc: - error "Error erasure decoding repairing manifest", - cid = manifest.treeCid, exc = exc.msg - return failure(exc.msg) - else: - without indexer =? - manifest.verifiableStrategy.init(0, manifest.blocksCount - 1, manifest.numSlots).catch, - err: - trace "Unable to create indexing strategy from protected manifest", err = err.msg - return failure(err) - - without blksIter =? indexer.getIndices(slotIdx.int).catch, err: - trace "Unable to get indices from strategy", err = err.msg - return failure(err) - - if err =? ( - await self.fetchBatched(manifest.treeCid, blksIter, onBatch = updateExpiry) - ).errorOption: - trace "Unable to fetch blocks", err = err.msg - return failure(err) - - without slotRoot =? (await builder.buildSlot(slotIdx.int)), err: - trace "Unable to build slot", err = err.msg - return failure(err) - - if cid =? slotRoot.toSlotCid() and cid != manifest.slotRoots[slotIdx]: - trace "Slot root mismatch", - manifest = manifest.slotRoots[slotIdx.int], recovered = slotRoot.toSlotCid() - return failure(newException(CodexError, "Slot root mismatch")) - - trace "Slot successfully retrieved and reconstructed" - - return success() - -proc onProve( - self: CodexNodeRef, slot: Slot, challenge: ProofChallenge -): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - ## Generats a proof for a given slot and challenge - ## - - let - cidStr = $slot.request.content.cid - slotIdx = slot.slotIndex - - logScope: - cid = cidStr - slot = slotIdx - challenge = challenge - - trace "Received proof challenge" - - if prover =? self.prover: - trace "Prover enabled" - - without cid =? Cid.init(cidStr).mapFailure, err: - error "Unable to parse Cid", cid, err = err.msg - return failure(err) - - without manifest =? await self.fetchManifest(cid), err: - error "Unable to fetch manifest for cid", err = err.msg - return failure(err) - - when defined(verify_circuit): - without (inputs, proof) =? await prover.prove(slotIdx.int, manifest, challenge), - err: - error "Unable to generate proof", err = err.msg - return failure(err) - - without checked =? await prover.verify(proof, inputs), err: - error "Unable to verify proof", err = err.msg - return failure(err) - - if not checked: - error "Proof verification failed" - return failure("Proof verification failed") - - trace "Proof verified successfully" - else: - without (_, proof) =? await prover.prove(slotIdx.int, manifest, challenge), err: - error "Unable to generate proof", err = err.msg - return failure(err) - - let groth16Proof = proof.toGroth16Proof() - trace "Proof generated successfully", groth16Proof - - success groth16Proof - else: - warn "Prover not enabled" - failure "Prover not enabled" - proc onExpiryUpdate( self: CodexNodeRef, rootCid: Cid, expiry: SecondsSince1970 ): Future[?!void] {.async: (raises: [CancelledError]).} = return await self.updateExpiry(rootCid, expiry) -proc onClear(self: CodexNodeRef, request: StorageRequest, slotIndex: uint64) = - # TODO: remove data from local storage - discard - proc start*(self: CodexNodeRef) {.async.} = if not self.engine.isNil: await self.engine.start() @@ -829,57 +517,6 @@ proc start*(self: CodexNodeRef) {.async.} = if not self.clock.isNil: await self.clock.start() - if hostContracts =? self.contracts.host: - hostContracts.sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing: bool = false, - ): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} = - self.onStore(request, expiry, slot, onBatch, isRepairing) - - hostContracts.sales.onExpiryUpdate = proc( - rootCid: Cid, expiry: SecondsSince1970 - ): Future[?!void] {.async: (raw: true, raises: [CancelledError]).} = - self.onExpiryUpdate(rootCid, expiry) - - hostContracts.sales.onClear = proc(request: StorageRequest, slotIndex: uint64) = - # TODO: remove data from local storage - self.onClear(request, slotIndex) - - hostContracts.sales.onProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raw: true, raises: [CancelledError]).} = - # TODO: generate proof - self.onProve(slot, challenge) - - try: - await hostContracts.start() - except CancelledError as error: - raise error - except CatchableError as error: - error "Unable to start host contract interactions", error = error.msg - self.contracts.host = HostInteractions.none - - if clientContracts =? self.contracts.client: - try: - await clientContracts.start() - except CancelledError as error: - raise error - except CatchableError as error: - error "Unable to start client contract interactions: ", error = error.msg - self.contracts.client = ClientInteractions.none - - if validatorContracts =? self.contracts.validator: - try: - await validatorContracts.start() - except CancelledError as error: - raise error - except CatchableError as error: - error "Unable to start validator contract interactions: ", error = error.msg - self.contracts.validator = ValidatorInteractions.none - self.networkId = self.switch.peerInfo.peerId notice "Started Storage node", id = self.networkId, addrs = self.switch.peerInfo.addrs @@ -894,15 +531,6 @@ proc stop*(self: CodexNodeRef) {.async.} = if not self.discovery.isNil: await self.discovery.stop() - if clientContracts =? self.contracts.client: - await clientContracts.stop() - - if hostContracts =? self.contracts.host: - await hostContracts.stop() - - if validatorContracts =? self.contracts.validator: - await validatorContracts.stop() - if not self.clock.isNil: await self.clock.stop() @@ -917,8 +545,6 @@ proc new*( engine: BlockExcEngine, discovery: Discovery, taskpool: Taskpool, - prover = Prover.none, - contracts = Contracts.default, ): CodexNodeRef = ## Create new instance of a Codex self, call `start` to run it ## @@ -927,10 +553,8 @@ proc new*( switch: switch, networkStore: networkStore, engine: engine, - prover: prover, discovery: discovery, taskPool: taskpool, - contracts: contracts, trackedFutures: TrackedFutures(), ) diff --git a/codex/periods.nim b/codex/periods.nim deleted file mode 100644 index cbb860e2..00000000 --- a/codex/periods.nim +++ /dev/null @@ -1,17 +0,0 @@ -import pkg/stint - -type - Periodicity* = object - seconds*: uint64 - - Period* = uint64 - Timestamp* = uint64 - -func periodOf*(periodicity: Periodicity, timestamp: Timestamp): Period = - timestamp div periodicity.seconds - -func periodStart*(periodicity: Periodicity, period: Period): Timestamp = - period * periodicity.seconds - -func periodEnd*(periodicity: Periodicity, period: Period): Timestamp = - periodicity.periodStart(period + 1) diff --git a/codex/purchasing.nim b/codex/purchasing.nim deleted file mode 100644 index 25a35137..00000000 --- a/codex/purchasing.nim +++ /dev/null @@ -1,74 +0,0 @@ -import std/tables -import pkg/stint -import pkg/chronos -import pkg/questionable -import pkg/nimcrypto -import ./market -import ./clock -import ./purchasing/purchase - -export questionable -export chronos -export market -export purchase - -type - Purchasing* = ref object - market*: Market - clock: Clock - purchases: Table[PurchaseId, Purchase] - proofProbability*: UInt256 - - PurchaseTimeout* = Timeout - -const DefaultProofProbability = 100.u256 - -proc new*(_: type Purchasing, market: Market, clock: Clock): Purchasing = - Purchasing(market: market, clock: clock, proofProbability: DefaultProofProbability) - -proc load*(purchasing: Purchasing) {.async.} = - let market = purchasing.market - let requestIds = await market.myRequests() - for requestId in requestIds: - let purchase = Purchase.new(requestId, purchasing.market, purchasing.clock) - purchase.load() - purchasing.purchases[purchase.id] = purchase - -proc start*(purchasing: Purchasing) {.async.} = - await purchasing.load() - -proc stop*(purchasing: Purchasing) {.async.} = - discard - -proc populate*( - purchasing: Purchasing, request: StorageRequest -): Future[StorageRequest] {.async.} = - result = request - if result.ask.proofProbability == 0.u256: - result.ask.proofProbability = purchasing.proofProbability - if result.nonce == Nonce.default: - var id = result.nonce.toArray - doAssert randomBytes(id) == 32 - result.nonce = Nonce(id) - result.client = await purchasing.market.getSigner() - -proc purchase*( - purchasing: Purchasing, request: StorageRequest -): Future[Purchase] {.async.} = - let request = await purchasing.populate(request) - let purchase = Purchase.new(request, purchasing.market, purchasing.clock) - purchase.start() - purchasing.purchases[purchase.id] = purchase - return purchase - -func getPurchase*(purchasing: Purchasing, id: PurchaseId): ?Purchase = - if purchasing.purchases.hasKey(id): - some purchasing.purchases[id] - else: - none Purchase - -func getPurchaseIds*(purchasing: Purchasing): seq[PurchaseId] = - var pIds: seq[PurchaseId] = @[] - for key in purchasing.purchases.keys: - pIds.add(key) - return pIds diff --git a/codex/purchasing/purchase.nim b/codex/purchasing/purchase.nim deleted file mode 100644 index 7c16c28c..00000000 --- a/codex/purchasing/purchase.nim +++ /dev/null @@ -1,74 +0,0 @@ -import ./statemachine -import ./states/pending -import ./states/unknown -import ./purchaseid - -# Purchase is implemented as a state machine. -# -# It can either be a new (pending) purchase that still needs to be submitted -# on-chain, or it is a purchase that was previously submitted on-chain, and -# we're just restoring its (unknown) state after a node restart. -# -# | -# v -# ------------------------- unknown -# | / / -# v v / -# pending ----> submitted ----> started ---------> finished <----/ -# \ \ / -# \ ------------> failed <----/ -# \ / -# --> cancelled <----------------------- - -export Purchase -export purchaseid -export statemachine - -func new*( - _: type Purchase, requestId: RequestId, market: Market, clock: Clock -): Purchase = - ## create a new instance of a Purchase - ## - var purchase = Purchase.new() - {.cast(noSideEffect).}: - purchase.future = newFuture[void]() - purchase.requestId = requestId - purchase.market = market - purchase.clock = clock - - return purchase - -func new*( - _: type Purchase, request: StorageRequest, market: Market, clock: Clock -): Purchase = - ## Create a new purchase using the given market and clock - let purchase = Purchase.new(request.id, market, clock) - purchase.request = some request - return purchase - -proc start*(purchase: Purchase) = - purchase.start(PurchasePending()) - -proc load*(purchase: Purchase) = - purchase.start(PurchaseUnknown()) - -proc wait*(purchase: Purchase) {.async.} = - await purchase.future - -func id*(purchase: Purchase): PurchaseId = - PurchaseId(purchase.requestId) - -func finished*(purchase: Purchase): bool = - purchase.future.finished - -func error*(purchase: Purchase): ?(ref CatchableError) = - if purchase.future.failed: - some purchase.future.error - else: - none (ref CatchableError) - -func state*(purchase: Purchase): ?string = - proc description(state: State): string = - $state - - purchase.query(description) diff --git a/codex/purchasing/purchaseid.nim b/codex/purchasing/purchaseid.nim deleted file mode 100644 index 965b0839..00000000 --- a/codex/purchasing/purchaseid.nim +++ /dev/null @@ -1,14 +0,0 @@ -import std/hashes -import ../logutils - -type PurchaseId* = distinct array[32, byte] - -logutils.formatIt(LogFormat.textLines, PurchaseId): - it.short0xHexLog -logutils.formatIt(LogFormat.json, PurchaseId): - it.to0xHexLog - -proc hash*(x: PurchaseId): Hash {.borrow.} -proc `==`*(x, y: PurchaseId): bool {.borrow.} -proc toHex*(x: PurchaseId): string = - array[32, byte](x).toHex diff --git a/codex/purchasing/statemachine.nim b/codex/purchasing/statemachine.nim deleted file mode 100644 index 20a63783..00000000 --- a/codex/purchasing/statemachine.nim +++ /dev/null @@ -1,19 +0,0 @@ -import ../utils/asyncstatemachine -import ../market -import ../clock -import ../errors - -export market -export clock -export asyncstatemachine - -type - Purchase* = ref object of Machine - future*: Future[void] - market*: Market - clock*: Clock - requestId*: RequestId - request*: ?StorageRequest - - PurchaseState* = ref object of State - PurchaseError* = object of CodexError diff --git a/codex/purchasing/states/cancelled.nim b/codex/purchasing/states/cancelled.nim deleted file mode 100644 index 5aeeceac..00000000 --- a/codex/purchasing/states/cancelled.nim +++ /dev/null @@ -1,35 +0,0 @@ -import pkg/metrics - -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ./error - -declareCounter(codex_purchases_cancelled, "codex purchases cancelled") - -logScope: - topics = "marketplace purchases cancelled" - -type PurchaseCancelled* = ref object of PurchaseState - -method `$`*(state: PurchaseCancelled): string = - "cancelled" - -method run*( - state: PurchaseCancelled, machine: Machine -): Future[?State] {.async: (raises: []).} = - codex_purchases_cancelled.inc() - let purchase = Purchase(machine) - - try: - warn "Request cancelled, withdrawing remaining funds", - requestId = purchase.requestId - await purchase.market.withdrawFunds(purchase.requestId) - - let error = newException(Timeout, "Purchase cancelled due to timeout") - purchase.future.fail(error) - except CancelledError as e: - trace "PurchaseCancelled.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during PurchaseCancelled.run", error = e.msgDetail - return some State(PurchaseErrored(error: e)) diff --git a/codex/purchasing/states/error.nim b/codex/purchasing/states/error.nim deleted file mode 100644 index afa9f54f..00000000 --- a/codex/purchasing/states/error.nim +++ /dev/null @@ -1,26 +0,0 @@ -import pkg/metrics -import ../statemachine -import ../../utils/exceptions -import ../../logutils - -declareCounter(codex_purchases_error, "codex purchases error") - -logScope: - topics = "marketplace purchases errored" - -type PurchaseErrored* = ref object of PurchaseState - error*: ref CatchableError - -method `$`*(state: PurchaseErrored): string = - "errored" - -method run*( - state: PurchaseErrored, machine: Machine -): Future[?State] {.async: (raises: []).} = - codex_purchases_error.inc() - let purchase = Purchase(machine) - - error "Purchasing error", - error = state.error.msgDetail, requestId = purchase.requestId - - purchase.future.fail(state.error) diff --git a/codex/purchasing/states/failed.nim b/codex/purchasing/states/failed.nim deleted file mode 100644 index 1f6be74f..00000000 --- a/codex/purchasing/states/failed.nim +++ /dev/null @@ -1,30 +0,0 @@ -import pkg/metrics -import ../statemachine -import ../../logutils -import ../../utils/exceptions -import ./error - -declareCounter(codex_purchases_failed, "codex purchases failed") - -type PurchaseFailed* = ref object of PurchaseState - -method `$`*(state: PurchaseFailed): string = - "failed" - -method run*( - state: PurchaseFailed, machine: Machine -): Future[?State] {.async: (raises: []).} = - codex_purchases_failed.inc() - let purchase = Purchase(machine) - - try: - warn "Request failed, withdrawing remaining funds", requestId = purchase.requestId - await purchase.market.withdrawFunds(purchase.requestId) - except CancelledError as e: - trace "PurchaseFailed.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during PurchaseFailed.run", error = e.msgDetail - return some State(PurchaseErrored(error: e)) - - let error = newException(PurchaseError, "Purchase failed") - return some State(PurchaseErrored(error: error)) diff --git a/codex/purchasing/states/finished.nim b/codex/purchasing/states/finished.nim deleted file mode 100644 index bb7a726d..00000000 --- a/codex/purchasing/states/finished.nim +++ /dev/null @@ -1,33 +0,0 @@ -import pkg/metrics - -import ../statemachine -import ../../utils/exceptions -import ../../logutils -import ./error - -declareCounter(codex_purchases_finished, "codex purchases finished") - -logScope: - topics = "marketplace purchases finished" - -type PurchaseFinished* = ref object of PurchaseState - -method `$`*(state: PurchaseFinished): string = - "finished" - -method run*( - state: PurchaseFinished, machine: Machine -): Future[?State] {.async: (raises: []).} = - codex_purchases_finished.inc() - let purchase = Purchase(machine) - try: - info "Purchase finished, withdrawing remaining funds", - requestId = purchase.requestId - await purchase.market.withdrawFunds(purchase.requestId) - - purchase.future.complete() - except CancelledError as e: - trace "PurchaseFinished.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during PurchaseFinished.run", error = e.msgDetail - return some State(PurchaseErrored(error: e)) diff --git a/codex/purchasing/states/pending.nim b/codex/purchasing/states/pending.nim deleted file mode 100644 index 1472a63e..00000000 --- a/codex/purchasing/states/pending.nim +++ /dev/null @@ -1,28 +0,0 @@ -import pkg/metrics -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ./submitted -import ./error - -declareCounter(codex_purchases_pending, "codex purchases pending") - -type PurchasePending* = ref object of PurchaseState - -method `$`*(state: PurchasePending): string = - "pending" - -method run*( - state: PurchasePending, machine: Machine -): Future[?State] {.async: (raises: []).} = - codex_purchases_pending.inc() - let purchase = Purchase(machine) - try: - let request = !purchase.request - await purchase.market.requestStorage(request) - return some State(PurchaseSubmitted()) - except CancelledError as e: - trace "PurchasePending.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during PurchasePending.run", error = e.msgDetail - return some State(PurchaseErrored(error: e)) diff --git a/codex/purchasing/states/started.nim b/codex/purchasing/states/started.nim deleted file mode 100644 index e93d7013..00000000 --- a/codex/purchasing/states/started.nim +++ /dev/null @@ -1,54 +0,0 @@ -import pkg/metrics - -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ./finished -import ./failed -import ./error - -declareCounter(codex_purchases_started, "codex purchases started") - -logScope: - topics = "marketplace purchases started" - -type PurchaseStarted* = ref object of PurchaseState - -method `$`*(state: PurchaseStarted): string = - "started" - -method run*( - state: PurchaseStarted, machine: Machine -): Future[?State] {.async: (raises: []).} = - codex_purchases_started.inc() - let purchase = Purchase(machine) - - let clock = purchase.clock - let market = purchase.market - info "All required slots filled, purchase started", requestId = purchase.requestId - - let failed = newFuture[void]() - proc callback(_: RequestId) = - failed.complete() - - var ended: Future[void] - try: - let subscription = await market.subscribeRequestFailed(purchase.requestId, callback) - - # Ensure that we're past the request end by waiting an additional second - ended = clock.waitUntil((await market.getRequestEnd(purchase.requestId)) + 1) - let fut = await one(ended, failed) - await subscription.unsubscribe() - if fut.id == failed.id: - ended.cancelSoon() - return some State(PurchaseFailed()) - else: - failed.cancelSoon() - return some State(PurchaseFinished()) - except CancelledError as e: - ended.cancelSoon() - failed.cancelSoon() - trace "PurchaseStarted.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during PurchaseStarted.run", error = e.msgDetail - return some State(PurchaseErrored(error: e)) diff --git a/codex/purchasing/states/submitted.nim b/codex/purchasing/states/submitted.nim deleted file mode 100644 index 96d384a4..00000000 --- a/codex/purchasing/states/submitted.nim +++ /dev/null @@ -1,56 +0,0 @@ -import pkg/metrics - -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ./started -import ./cancelled -import ./error - -logScope: - topics = "marketplace purchases submitted" - -declareCounter(codex_purchases_submitted, "codex purchases submitted") - -type PurchaseSubmitted* = ref object of PurchaseState - -method `$`*(state: PurchaseSubmitted): string = - "submitted" - -method run*( - state: PurchaseSubmitted, machine: Machine -): Future[?State] {.async: (raises: []).} = - codex_purchases_submitted.inc() - let purchase = Purchase(machine) - let request = !purchase.request - let market = purchase.market - let clock = purchase.clock - - info "Request submitted, waiting for slots to be filled", - requestId = purchase.requestId - - proc wait() {.async.} = - let done = newAsyncEvent() - proc callback(_: RequestId) = - done.fire() - - let subscription = await market.subscribeFulfillment(request.id, callback) - await done.wait() - await subscription.unsubscribe() - - proc withTimeout(future: Future[void]) {.async.} = - let expiry = (await market.requestExpiresAt(request.id)) + 1 - trace "waiting for request fulfillment or expiry", expiry - await future.withTimeout(clock, expiry) - - try: - await wait().withTimeout() - except Timeout: - return some State(PurchaseCancelled()) - except CancelledError as e: - trace "PurchaseSubmitted.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during PurchaseSubmitted.run", error = e.msgDetail - return some State(PurchaseErrored(error: e)) - - return some State(PurchaseStarted()) diff --git a/codex/purchasing/states/unknown.nim b/codex/purchasing/states/unknown.nim deleted file mode 100644 index 8c2bff48..00000000 --- a/codex/purchasing/states/unknown.nim +++ /dev/null @@ -1,44 +0,0 @@ -import pkg/metrics -import ../../utils/exceptions -import ../../logutils -import ../statemachine -import ./submitted -import ./started -import ./cancelled -import ./finished -import ./failed -import ./error - -declareCounter(codex_purchases_unknown, "codex purchases unknown") - -type PurchaseUnknown* = ref object of PurchaseState - -method `$`*(state: PurchaseUnknown): string = - "unknown" - -method run*( - state: PurchaseUnknown, machine: Machine -): Future[?State] {.async: (raises: []).} = - try: - codex_purchases_unknown.inc() - let purchase = Purchase(machine) - if (request =? await purchase.market.getRequest(purchase.requestId)) and - (requestState =? await purchase.market.requestState(purchase.requestId)): - purchase.request = some request - - case requestState - of RequestState.New: - return some State(PurchaseSubmitted()) - of RequestState.Started: - return some State(PurchaseStarted()) - of RequestState.Cancelled: - return some State(PurchaseCancelled()) - of RequestState.Finished: - return some State(PurchaseFinished()) - of RequestState.Failed: - return some State(PurchaseFailed()) - except CancelledError as e: - trace "PurchaseUnknown.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during PurchaseUnknown.run", error = e.msgDetail - return some State(PurchaseErrored(error: e)) diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 5650f230..8393a85b 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -30,8 +30,6 @@ import ../logutils import ../node import ../blocktype import ../conf -import ../contracts -import ../erasure/erasure import ../manifest import ../streams/asyncstreamwrapper import ../stores @@ -116,9 +114,7 @@ proc retrieveCid( # For erasure-coded datasets, we need to return the _original_ length; i.e., # the length of the non-erasure-coded dataset, as that's what we will be # returning to the client. - let contentLength = - if manifest.protected: manifest.originalDatasetSize else: manifest.datasetSize - resp.setHeader("Content-Length", $(contentLength.int)) + resp.setHeader("Content-Length", $(manifest.datasetSize.int)) await resp.prepare(HttpResponseStreamType.Plain) @@ -388,459 +384,6 @@ proc initDataApi(node: CodexNodeRef, repoStore: RepoStore, router: var RestRoute ) return RestApiResponse.response($json, contentType = "application/json") -proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = - let allowedOrigin = router.allowedOrigin - - router.api(MethodGet, "/api/storage/v1/sales/slots") do() -> RestApiResponse: - var headers = buildCorsHeaders("GET", allowedOrigin) - - ## Returns active slots for the host - try: - without contracts =? node.contracts.host: - return RestApiResponse.error( - Http503, "Persistence is not enabled", headers = headers - ) - - let json = %(await contracts.sales.mySlots()) - return RestApiResponse.response( - $json, contentType = "application/json", headers = headers - ) - except CatchableError as exc: - trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500, headers = headers) - - router.api(MethodGet, "/api/storage/v1/sales/slots/{slotId}") do( - slotId: SlotId - ) -> RestApiResponse: - ## Returns active slot with id {slotId} for the host. Returns 404 if the - ## slot is not active for the host. - var headers = buildCorsHeaders("GET", allowedOrigin) - - without contracts =? node.contracts.host: - return - RestApiResponse.error(Http503, "Persistence is not enabled", headers = headers) - - without slotId =? slotId.tryGet.catch, error: - return RestApiResponse.error(Http400, error.msg, headers = headers) - - without agent =? await contracts.sales.activeSale(slotId): - return - RestApiResponse.error(Http404, "Provider not filling slot", headers = headers) - - let restAgent = RestSalesAgent( - state: agent.state() |? "none", - slotIndex: agent.data.slotIndex, - requestId: agent.data.requestId, - request: agent.data.request, - reservation: agent.data.reservation, - ) - - return RestApiResponse.response( - restAgent.toJson, contentType = "application/json", headers = headers - ) - - router.api(MethodGet, "/api/storage/v1/sales/availability") do() -> RestApiResponse: - ## Returns storage that is for sale - var headers = buildCorsHeaders("GET", allowedOrigin) - - try: - without contracts =? node.contracts.host: - return RestApiResponse.error( - Http503, "Persistence is not enabled", headers = headers - ) - - without avails =? (await contracts.sales.context.reservations.all(Availability)), - err: - return RestApiResponse.error(Http500, err.msg, headers = headers) - - let json = %avails - return RestApiResponse.response( - $json, contentType = "application/json", headers = headers - ) - except CatchableError as exc: - trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500, headers = headers) - - router.rawApi(MethodPost, "/api/storage/v1/sales/availability") do() -> RestApiResponse: - ## Add available storage to sell. - ## Every time Availability's offer finishes, its capacity is - ## returned to the availability. - ## - ## totalSize - size of available storage in bytes - ## duration - maximum time the storage should be sold for (in seconds) - ## minPricePerBytePerSecond - minimal price per byte paid (in amount of - ## tokens) to be matched against the request's pricePerBytePerSecond - ## totalCollateral - total collateral (in amount of - ## tokens) that can be distributed among matching requests - - var headers = buildCorsHeaders("POST", allowedOrigin) - - try: - without contracts =? node.contracts.host: - return RestApiResponse.error( - Http503, "Persistence is not enabled", headers = headers - ) - - let body = await request.getBody() - - without restAv =? RestAvailability.fromJson(body), error: - return RestApiResponse.error(Http400, error.msg, headers = headers) - - let reservations = contracts.sales.context.reservations - - if restAv.totalSize == 0: - return RestApiResponse.error( - Http422, "Total size must be larger then zero", headers = headers - ) - - if restAv.duration == 0: - return RestApiResponse.error( - Http422, "duration must be larger then zero", headers = headers - ) - - if restAv.minPricePerBytePerSecond == 0: - return RestApiResponse.error( - Http422, - "minPricePerBytePerSecond must be larger then zero", - headers = headers, - ) - - if restAv.totalCollateral == 0: - return RestApiResponse.error( - Http422, "totalCollateral must be larger then zero", headers = headers - ) - - if not reservations.hasAvailable(restAv.totalSize): - return - RestApiResponse.error(Http422, "Not enough storage quota", headers = headers) - - without availability =? ( - await reservations.createAvailability( - restAv.totalSize, - restAv.duration, - restAv.minPricePerBytePerSecond, - restAv.totalCollateral, - enabled = restAv.enabled |? true, - until = restAv.until |? 0, - ) - ), error: - if error of CancelledError: - raise error - if error of UntilOutOfBoundsError: - return RestApiResponse.error(Http422, error.msg) - - return RestApiResponse.error(Http500, error.msg, headers = headers) - - return RestApiResponse.response( - availability.toJson, - Http201, - contentType = "application/json", - headers = headers, - ) - except CatchableError as exc: - trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500, headers = headers) - - router.api(MethodOptions, "/api/storage/v1/sales/availability/{id}") do( - id: AvailabilityId, resp: HttpResponseRef - ) -> RestApiResponse: - if corsOrigin =? allowedOrigin: - resp.setCorsHeaders("PATCH", corsOrigin) - - resp.status = Http204 - await resp.sendBody("") - - router.rawApi(MethodPatch, "/api/storage/v1/sales/availability/{id}") do( - id: AvailabilityId - ) -> RestApiResponse: - ## Updates Availability. - ## The new parameters will be only considered for new requests. - ## Existing Requests linked to this Availability will continue as is. - ## - ## totalSize - size of available storage in bytes. - ## When decreasing the size, then lower limit is - ## the currently `totalSize - freeSize`. - ## duration - maximum time the storage should be sold for (in seconds) - ## minPricePerBytePerSecond - minimal price per byte paid (in amount of - ## tokens) to be matched against the request's pricePerBytePerSecond - ## totalCollateral - total collateral (in amount of - ## tokens) that can be distributed among matching requests - - try: - without contracts =? node.contracts.host: - return RestApiResponse.error(Http503, "Persistence is not enabled") - - without id =? id.tryGet.catch, error: - return RestApiResponse.error(Http400, error.msg) - without keyId =? id.key.tryGet.catch, error: - return RestApiResponse.error(Http400, error.msg) - - let - body = await request.getBody() - reservations = contracts.sales.context.reservations - - type OptRestAvailability = Optionalize(RestAvailability) - without restAv =? OptRestAvailability.fromJson(body), error: - return RestApiResponse.error(Http400, error.msg) - - without availability =? (await reservations.get(keyId, Availability)), error: - if error of NotExistsError: - return RestApiResponse.error(Http404, "Availability not found") - - return RestApiResponse.error(Http500, error.msg) - - if isSome restAv.freeSize: - return RestApiResponse.error(Http422, "Updating freeSize is not allowed") - - if size =? restAv.totalSize: - if size == 0: - return RestApiResponse.error(Http422, "Total size must be larger then zero") - - # we don't allow lowering the totalSize bellow currently utilized size - if size < (availability.totalSize - availability.freeSize): - return RestApiResponse.error( - Http422, - "New totalSize must be larger then current totalSize - freeSize, which is currently: " & - $(availability.totalSize - availability.freeSize), - ) - - if not reservations.hasAvailable(size): - return RestApiResponse.error(Http422, "Not enough storage quota") - - availability.freeSize += size - availability.totalSize - availability.totalSize = size - - if duration =? restAv.duration: - availability.duration = duration - - if minPricePerBytePerSecond =? restAv.minPricePerBytePerSecond: - availability.minPricePerBytePerSecond = minPricePerBytePerSecond - - if totalCollateral =? restAv.totalCollateral: - availability.totalCollateral = totalCollateral - - if until =? restAv.until: - availability.until = until - - if enabled =? restAv.enabled: - availability.enabled = enabled - - if err =? (await reservations.update(availability)).errorOption: - if err of CancelledError: - raise err - if err of UntilOutOfBoundsError: - return RestApiResponse.error(Http422, err.msg) - else: - return RestApiResponse.error(Http500, err.msg) - - return RestApiResponse.response(Http204) - except CatchableError as exc: - trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500) - - router.rawApi(MethodGet, "/api/storage/v1/sales/availability/{id}/reservations") do( - id: AvailabilityId - ) -> RestApiResponse: - ## Gets Availability's reservations. - var headers = buildCorsHeaders("GET", allowedOrigin) - - try: - without contracts =? node.contracts.host: - return RestApiResponse.error( - Http503, "Persistence is not enabled", headers = headers - ) - - without id =? id.tryGet.catch, error: - return RestApiResponse.error(Http400, error.msg, headers = headers) - without keyId =? id.key.tryGet.catch, error: - return RestApiResponse.error(Http400, error.msg, headers = headers) - - let reservations = contracts.sales.context.reservations - let market = contracts.sales.context.market - - if error =? (await reservations.get(keyId, Availability)).errorOption: - if error of NotExistsError: - return - RestApiResponse.error(Http404, "Availability not found", headers = headers) - else: - return RestApiResponse.error(Http500, error.msg, headers = headers) - - without availabilitysReservations =? (await reservations.all(Reservation, id)), - err: - return RestApiResponse.error(Http500, err.msg, headers = headers) - - # TODO: Expand this structure with information about the linked StorageRequest not only RequestID - return RestApiResponse.response( - availabilitysReservations.toJson, - contentType = "application/json", - headers = headers, - ) - except CatchableError as exc: - trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500, headers = headers) - -proc initPurchasingApi(node: CodexNodeRef, router: var RestRouter) = - let allowedOrigin = router.allowedOrigin - - router.rawApi(MethodPost, "/api/storage/v1/storage/request/{cid}") do( - cid: Cid - ) -> RestApiResponse: - var headers = buildCorsHeaders("POST", allowedOrigin) - - ## Create a request for storage - ## - ## cid - the cid of a previously uploaded dataset - ## duration - the duration of the request in seconds - ## proofProbability - how often storage proofs are required - ## pricePerBytePerSecond - the amount of tokens paid per byte per second to hosts the client is willing to pay - ## expiry - specifies threshold in seconds from now when the request expires if the Request does not find requested amount of nodes to host the data - ## nodes - number of nodes the content should be stored on - ## tolerance - allowed number of nodes that can be lost before content is lost - ## colateralPerByte - requested collateral per byte from hosts when they fill slot - try: - without contracts =? node.contracts.client: - return RestApiResponse.error( - Http503, "Persistence is not enabled", headers = headers - ) - - without cid =? cid.tryGet.catch, error: - return RestApiResponse.error(Http400, error.msg, headers = headers) - - let body = await request.getBody() - - without params =? StorageRequestParams.fromJson(body), error: - return RestApiResponse.error(Http400, error.msg, headers = headers) - - let expiry = params.expiry - - if expiry <= 0 or expiry >= params.duration: - return RestApiResponse.error( - Http422, - "Expiry must be greater than zero and less than the request's duration", - headers = headers, - ) - - if params.proofProbability <= 0: - return RestApiResponse.error( - Http422, "Proof probability must be greater than zero", headers = headers - ) - - if params.collateralPerByte <= 0: - return RestApiResponse.error( - Http422, "Collateral per byte must be greater than zero", headers = headers - ) - - if params.pricePerBytePerSecond <= 0: - return RestApiResponse.error( - Http422, - "Price per byte per second must be greater than zero", - headers = headers, - ) - - let requestDurationLimit = await contracts.purchasing.market.requestDurationLimit - if params.duration > requestDurationLimit: - return RestApiResponse.error( - Http422, - "Duration exceeds limit of " & $requestDurationLimit & " seconds", - headers = headers, - ) - - let nodes = params.nodes |? 3 - let tolerance = params.tolerance |? 1 - - if tolerance == 0: - return RestApiResponse.error( - Http422, "Tolerance needs to be bigger then zero", headers = headers - ) - - # prevent underflow - if tolerance > nodes: - return RestApiResponse.error( - Http422, - "Invalid parameters: `tolerance` cannot be greater than `nodes`", - headers = headers, - ) - - let ecK = nodes - tolerance - let ecM = tolerance # for readability - - # ensure leopard constrainst of 1 < K ≥ M - if ecK <= 1 or ecK < ecM: - return RestApiResponse.error( - Http422, - "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`", - headers = headers, - ) - - without purchaseId =? - await node.requestStorage( - cid, params.duration, params.proofProbability, nodes, tolerance, - params.pricePerBytePerSecond, params.collateralPerByte, expiry, - ), error: - if error of InsufficientBlocksError: - return RestApiResponse.error( - Http422, - "Dataset too small for erasure parameters, need at least " & - $(ref InsufficientBlocksError)(error).minSize.int & " bytes", - headers = headers, - ) - - return RestApiResponse.error(Http500, error.msg, headers = headers) - - return RestApiResponse.response(purchaseId.toHex) - except CatchableError as exc: - trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500, headers = headers) - - router.api(MethodGet, "/api/storage/v1/storage/purchases/{id}") do( - id: PurchaseId - ) -> RestApiResponse: - var headers = buildCorsHeaders("GET", allowedOrigin) - - try: - without contracts =? node.contracts.client: - return RestApiResponse.error( - Http503, "Persistence is not enabled", headers = headers - ) - - without id =? id.tryGet.catch, error: - return RestApiResponse.error(Http400, error.msg, headers = headers) - - without purchase =? contracts.purchasing.getPurchase(id): - return RestApiResponse.error(Http404, headers = headers) - - let json = - %RestPurchase( - state: purchase.state |? "none", - error: purchase.error .? msg, - request: purchase.request, - requestId: purchase.requestId, - ) - - return RestApiResponse.response( - $json, contentType = "application/json", headers = headers - ) - except CatchableError as exc: - trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500, headers = headers) - - router.api(MethodGet, "/api/storage/v1/storage/purchases") do() -> RestApiResponse: - var headers = buildCorsHeaders("GET", allowedOrigin) - - try: - without contracts =? node.contracts.client: - return RestApiResponse.error( - Http503, "Persistence is not enabled", headers = headers - ) - - let purchaseIds = contracts.purchasing.getPurchaseIds() - return RestApiResponse.response( - $ %purchaseIds, contentType = "application/json", headers = headers - ) - except CatchableError as exc: - trace "Excepting processing request", exc = exc.msg - return RestApiResponse.error(Http500, headers = headers) - proc initNodeApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = let allowedOrigin = router.allowedOrigin @@ -949,7 +492,6 @@ proc initDebugApi(node: CodexNodeRef, conf: CodexConf, router: var RestRouter) = "storage": { "version": $codexVersion, "revision": $codexRevision, - "contracts": $codexContractsRevision, }, } @@ -1016,8 +558,6 @@ proc initRestApi*( var router = RestRouter.init(validate, corsAllowedOrigin) initDataApi(node, repoStore, router) - initSalesApi(node, router) - initPurchasingApi(node, router) initNodeApi(node, conf, router) initDebugApi(node, conf, router) diff --git a/codex/rest/coders.nim b/codex/rest/coders.nim index 3f9388d6..7db72ca9 100644 --- a/codex/rest/coders.nim +++ b/codex/rest/coders.nim @@ -17,8 +17,6 @@ import pkg/stew/byteutils import pkg/results import pkg/stint -import ../sales -import ../purchasing import ../utils/stintutils proc encodeString*(cid: type Cid): Result[string, cstring] = @@ -82,11 +80,6 @@ proc decodeString*( except ValueError as e: err e.msg.cstring -proc decodeString*[T: PurchaseId | RequestId | Nonce | SlotId | AvailabilityId]( - _: type T, value: string -): Result[T, cstring] = - array[32, byte].decodeString(value).map(id => T(id)) - proc decodeString*(t: typedesc[string], value: string): Result[string, cstring] = ok(value) diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 1b9459c1..cfe4769c 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -3,8 +3,6 @@ import pkg/stew/byteutils import pkg/libp2p import pkg/codexdht/discv5/node as dn import pkg/codexdht/discv5/routing_table as rt -import ../sales -import ../purchasing import ../utils/json import ../manifest import ../units @@ -12,36 +10,6 @@ import ../units export json type - StorageRequestParams* = object - duration* {.serialize.}: uint64 - proofProbability* {.serialize.}: UInt256 - pricePerBytePerSecond* {.serialize.}: UInt256 - collateralPerByte* {.serialize.}: UInt256 - expiry* {.serialize.}: uint64 - nodes* {.serialize.}: ?uint - tolerance* {.serialize.}: ?uint - - RestPurchase* = object - requestId* {.serialize.}: RequestId - request* {.serialize.}: ?StorageRequest - state* {.serialize.}: string - error* {.serialize.}: ?string - - RestAvailability* = object - totalSize* {.serialize.}: uint64 - duration* {.serialize.}: uint64 - minPricePerBytePerSecond* {.serialize.}: UInt256 - totalCollateral* {.serialize.}: UInt256 - freeSize* {.serialize.}: ?uint64 - enabled* {.serialize.}: ?bool - until* {.serialize.}: ?SecondsSince1970 - - RestSalesAgent* = object - state* {.serialize.}: string - requestId* {.serialize.}: RequestId - slotIndex* {.serialize.}: uint64 - request* {.serialize.}: ?StorageRequest - reservation* {.serialize.}: ?Reservation RestContent* = object cid* {.serialize.}: Cid @@ -106,13 +74,5 @@ proc init*(_: type RestPeerRecord, peerRecord: PeerRecord): RestPeerRecord = proc init*(_: type RestNodeId, id: NodeId): RestNodeId = RestNodeId(id: id) -proc `%`*(obj: StorageRequest | Slot): JsonNode = - let jsonObj = newJObject() - for k, v in obj.fieldPairs: - jsonObj[k] = %v - jsonObj["id"] = %(obj.id) - - return jsonObj - proc `%`*(obj: RestNodeId): JsonNode = % $obj.id diff --git a/codex/sales.nim b/codex/sales.nim deleted file mode 100644 index bc4262f1..00000000 --- a/codex/sales.nim +++ /dev/null @@ -1,555 +0,0 @@ -import std/sequtils -import std/sugar -import pkg/questionable -import pkg/questionable/results -import pkg/stint -import pkg/datastore -import ./market -import ./clock -import ./stores -import ./contracts/requests -import ./contracts/marketplace -import ./logutils -import ./sales/salescontext -import ./sales/salesagent -import ./sales/statemachine -import ./sales/slotqueue -import ./sales/states/preparing -import ./sales/states/unknown -import ./utils/trackedfutures -import ./utils/exceptions - -## Sales holds a list of available storage that it may sell. -## -## When storage is requested on the market that matches availability, the Sales -## object will instruct the Logos Storage node to persist the requested data. Once the -## data has been persisted, it uploads a proof of storage to the market in an -## attempt to win a storage contract. -## -## Node Sales Market -## | | | -## | -- add availability --> | | -## | | <-- storage request --- | -## | <----- store data ------ | | -## | -----------------------> | | -## | | | -## | <----- prove data ---- | | -## | -----------------------> | | -## | | ---- storage proof ---> | - -export stint -export reservations -export salesagent -export salescontext - -logScope: - topics = "sales marketplace" - -type Sales* = ref object - context*: SalesContext - agents*: seq[SalesAgent] - running: bool - subscriptions: seq[market.Subscription] - trackedFutures: TrackedFutures - -proc `onStore=`*(sales: Sales, onStore: OnStore) = - sales.context.onStore = some onStore - -proc `onClear=`*(sales: Sales, onClear: OnClear) = - sales.context.onClear = some onClear - -proc `onSale=`*(sales: Sales, callback: OnSale) = - sales.context.onSale = some callback - -proc `onProve=`*(sales: Sales, callback: OnProve) = - sales.context.onProve = some callback - -proc `onExpiryUpdate=`*(sales: Sales, callback: OnExpiryUpdate) = - sales.context.onExpiryUpdate = some callback - -proc onStore*(sales: Sales): ?OnStore = - sales.context.onStore - -proc onClear*(sales: Sales): ?OnClear = - sales.context.onClear - -proc onSale*(sales: Sales): ?OnSale = - sales.context.onSale - -proc onProve*(sales: Sales): ?OnProve = - sales.context.onProve - -proc onExpiryUpdate*(sales: Sales): ?OnExpiryUpdate = - sales.context.onExpiryUpdate - -proc new*(_: type Sales, market: Market, clock: Clock, repo: RepoStore): Sales = - Sales.new(market, clock, repo, 0) - -proc new*( - _: type Sales, - market: Market, - clock: Clock, - repo: RepoStore, - simulateProofFailures: int, -): Sales = - let reservations = Reservations.new(repo) - Sales( - context: SalesContext( - market: market, - clock: clock, - reservations: reservations, - slotQueue: SlotQueue.new(), - simulateProofFailures: simulateProofFailures, - ), - trackedFutures: TrackedFutures.new(), - subscriptions: @[], - ) - -proc remove(sales: Sales, agent: SalesAgent) {.async: (raises: []).} = - await agent.stop() - - if sales.running: - sales.agents.keepItIf(it != agent) - -proc cleanUp( - sales: Sales, agent: SalesAgent, reprocessSlot: bool, returnedCollateral: ?UInt256 -) {.async: (raises: []).} = - let data = agent.data - - logScope: - topics = "sales cleanUp" - requestId = data.requestId - slotIndex = data.slotIndex - reservationId = data.reservation .? id |? ReservationId.default - availabilityId = data.reservation .? availabilityId |? AvailabilityId.default - - trace "cleaning up sales agent" - - # if reservation for the SalesAgent was not created, then it means - # that the cleanUp was called before the sales process really started, so - # there are not really any bytes to be returned - if request =? data.request and reservation =? data.reservation: - if returnErr =? ( - await noCancel sales.context.reservations.returnBytesToAvailability( - reservation.availabilityId, reservation.id, request.ask.slotSize - ) - ).errorOption: - error "failure returning bytes", - error = returnErr.msg, bytes = request.ask.slotSize - - # delete reservation and return reservation bytes back to the availability - if reservation =? data.reservation and - deleteErr =? ( - await noCancel sales.context.reservations.deleteReservation( - reservation.id, reservation.availabilityId, returnedCollateral - ) - ).errorOption: - error "failure deleting reservation", error = deleteErr.msg - - # Re-add items back into the queue to prevent small availabilities from - # draining the queue. Seen items will be ordered last. - if reprocessSlot and request =? data.request and var item =? agent.data.slotQueueItem: - let queue = sales.context.slotQueue - item.seen = true - trace "pushing ignored item to queue, marked as seen" - if err =? queue.push(item).errorOption: - error "failed to readd slot to queue", errorType = $(type err), error = err.msg - - let fut = sales.remove(agent) - sales.trackedFutures.track(fut) - -proc filled(sales: Sales, request: StorageRequest, slotIndex: uint64) = - if onSale =? sales.context.onSale: - onSale(request, slotIndex) - -proc processSlot( - sales: Sales, item: SlotQueueItem -) {.async: (raises: [CancelledError]).} = - debug "Processing slot from queue", requestId = item.requestId, slot = item.slotIndex - - let agent = newSalesAgent( - sales.context, item.requestId, item.slotIndex, none StorageRequest, some item - ) - - let completed = newAsyncEvent() - - agent.onCleanUp = proc( - reprocessSlot = false, returnedCollateral = UInt256.none - ) {.async: (raises: []).} = - trace "slot cleanup" - await sales.cleanUp(agent, reprocessSlot, returnedCollateral) - completed.fire() - - agent.onFilled = some proc(request: StorageRequest, slotIndex: uint64) = - trace "slot filled" - sales.filled(request, slotIndex) - completed.fire() - - agent.start(SalePreparing()) - sales.agents.add agent - - trace "waiting for slot processing to complete" - await completed.wait() - trace "slot processing completed" - -proc deleteInactiveReservations(sales: Sales, activeSlots: seq[Slot]) {.async.} = - let reservations = sales.context.reservations - without reservs =? await reservations.all(Reservation): - return - - let unused = reservs.filter( - r => ( - let slotId = slotId(r.requestId, r.slotIndex) - not activeSlots.any(slot => slot.id == slotId) - ) - ) - - if unused.len == 0: - return - - info "Found unused reservations for deletion", unused = unused.len - - for reservation in unused: - logScope: - reservationId = reservation.id - availabilityId = reservation.availabilityId - - if err =? ( - await reservations.deleteReservation(reservation.id, reservation.availabilityId) - ).errorOption: - error "Failed to delete unused reservation", error = err.msg - else: - trace "Deleted unused reservation" - -proc mySlots*(sales: Sales): Future[seq[Slot]] {.async.} = - let market = sales.context.market - let slotIds = await market.mySlots() - var slots: seq[Slot] = @[] - - info "Loading active slots", slotsCount = len(slots) - for slotId in slotIds: - if slot =? (await market.getActiveSlot(slotId)): - slots.add slot - - return slots - -proc activeSale*(sales: Sales, slotId: SlotId): Future[?SalesAgent] {.async.} = - for agent in sales.agents: - if slotId(agent.data.requestId, agent.data.slotIndex) == slotId: - return some agent - - return none SalesAgent - -proc load*(sales: Sales) {.async.} = - let activeSlots = await sales.mySlots() - - await sales.deleteInactiveReservations(activeSlots) - - for slot in activeSlots: - let agent = - newSalesAgent(sales.context, slot.request.id, slot.slotIndex, some slot.request) - - agent.onCleanUp = proc( - reprocessSlot = false, returnedCollateral = UInt256.none - ) {.async: (raises: []).} = - await sales.cleanUp(agent, reprocessSlot, returnedCollateral) - - # There is no need to assign agent.onFilled as slots loaded from `mySlots` - # are inherently already filled and so assigning agent.onFilled would be - # superfluous. - - agent.start(SaleUnknown()) - sales.agents.add agent - -proc OnAvailabilitySaved( - sales: Sales, availability: Availability -) {.async: (raises: []).} = - ## When availabilities are modified or added, the queue should be unpaused if - ## it was paused and any slots in the queue should have their `seen` flag - ## cleared. - let queue = sales.context.slotQueue - - queue.clearSeenFlags() - if queue.paused: - trace "unpausing queue after new availability added" - queue.unpause() - -proc onStorageRequested( - sales: Sales, requestId: RequestId, ask: StorageAsk, expiry: uint64 -) {.raises: [].} = - logScope: - topics = "marketplace sales onStorageRequested" - requestId - slots = ask.slots - expiry - - let slotQueue = sales.context.slotQueue - - trace "storage requested, adding slots to queue" - - let market = sales.context.market - - without collateral =? market.slotCollateral(ask.collateralPerSlot, SlotState.Free), - err: - error "Request failure, unable to calculate collateral", error = err.msg - return - - without items =? SlotQueueItem.init(requestId, ask, expiry, collateral).catch, err: - if err of SlotsOutOfRangeError: - warn "Too many slots, cannot add to queue" - else: - warn "Failed to create slot queue items from request", error = err.msg - return - - for item in items: - # continue on failure - if err =? slotQueue.push(item).errorOption: - if err of SlotQueueItemExistsError: - error "Failed to push item to queue becaue it already exists" - elif err of QueueNotRunningError: - warn "Failed to push item to queue becaue queue is not running" - else: - warn "Error adding request to SlotQueue", error = err.msg - -proc onSlotFreed(sales: Sales, requestId: RequestId, slotIndex: uint64) = - logScope: - topics = "marketplace sales onSlotFreed" - requestId - slotIndex - - trace "slot freed, adding to queue" - - proc addSlotToQueue() {.async: (raises: []).} = - let context = sales.context - let market = context.market - let queue = context.slotQueue - - try: - without request =? (await market.getRequest(requestId)), err: - error "unknown request in contract", error = err.msgDetail - return - - # Take the repairing state into consideration to calculate the collateral. - # This is particularly needed because it will affect the priority in the queue - # and we want to give the user the ability to tweak the parameters. - # Adding the repairing state directly in the queue priority calculation - # would not allow this flexibility. - without collateral =? - market.slotCollateral(request.ask.collateralPerSlot, SlotState.Repair), err: - error "Failed to add freed slot to queue: unable to calculate collateral", - error = err.msg - return - - if slotIndex > uint16.high.uint64: - error "Cannot cast slot index to uint16, value = ", slotIndex - return - - without slotQueueItem =? - SlotQueueItem.init(request, slotIndex.uint16, collateral = collateral).catch, - err: - warn "Too many slots, cannot add to queue", error = err.msgDetail - return - - if err =? queue.push(slotQueueItem).errorOption: - if err of SlotQueueItemExistsError: - error "Failed to push item to queue because it already exists", - error = err.msgDetail - elif err of QueueNotRunningError: - warn "Failed to push item to queue because queue is not running", - error = err.msgDetail - except CancelledError as e: - trace "sales.addSlotToQueue was cancelled" - - # We could get rid of this by adding the storage ask in the SlotFreed event, - # so we would not need to call getRequest to get the collateralPerSlot. - let fut = addSlotToQueue() - sales.trackedFutures.track(fut) - -proc subscribeRequested(sales: Sales) {.async.} = - let context = sales.context - let market = context.market - - proc onStorageRequested( - requestId: RequestId, ask: StorageAsk, expiry: uint64 - ) {.raises: [].} = - sales.onStorageRequested(requestId, ask, expiry) - - try: - let sub = await market.subscribeRequests(onStorageRequested) - sales.subscriptions.add(sub) - except CancelledError as error: - raise error - except CatchableError as e: - error "Unable to subscribe to storage request events", msg = e.msg - -proc subscribeCancellation(sales: Sales) {.async.} = - let context = sales.context - let market = context.market - let queue = context.slotQueue - - proc onCancelled(requestId: RequestId) = - trace "request cancelled (via contract RequestCancelled event), removing all request slots from queue" - queue.delete(requestId) - - try: - let sub = await market.subscribeRequestCancelled(onCancelled) - sales.subscriptions.add(sub) - except CancelledError as error: - raise error - except CatchableError as e: - error "Unable to subscribe to cancellation events", msg = e.msg - -proc subscribeFulfilled*(sales: Sales) {.async.} = - let context = sales.context - let market = context.market - let queue = context.slotQueue - - proc onFulfilled(requestId: RequestId) = - trace "request fulfilled, removing all request slots from queue" - queue.delete(requestId) - - for agent in sales.agents: - agent.onFulfilled(requestId) - - try: - let sub = await market.subscribeFulfillment(onFulfilled) - sales.subscriptions.add(sub) - except CancelledError as error: - raise error - except CatchableError as e: - error "Unable to subscribe to storage fulfilled events", msg = e.msg - -proc subscribeFailure(sales: Sales) {.async.} = - let context = sales.context - let market = context.market - let queue = context.slotQueue - - proc onFailed(requestId: RequestId) = - trace "request failed, removing all request slots from queue" - queue.delete(requestId) - - for agent in sales.agents: - agent.onFailed(requestId) - - try: - let sub = await market.subscribeRequestFailed(onFailed) - sales.subscriptions.add(sub) - except CancelledError as error: - raise error - except CatchableError as e: - error "Unable to subscribe to storage failure events", msg = e.msg - -proc subscribeSlotFilled(sales: Sales) {.async.} = - let context = sales.context - let market = context.market - let queue = context.slotQueue - - proc onSlotFilled(requestId: RequestId, slotIndex: uint64) = - if slotIndex > uint16.high.uint64: - error "Cannot cast slot index to uint16, value = ", slotIndex - return - - trace "slot filled, removing from slot queue", requestId, slotIndex - queue.delete(requestId, slotIndex.uint16) - - for agent in sales.agents: - agent.onSlotFilled(requestId, slotIndex) - - try: - let sub = await market.subscribeSlotFilled(onSlotFilled) - sales.subscriptions.add(sub) - except CancelledError as error: - raise error - except CatchableError as e: - error "Unable to subscribe to slot filled events", msg = e.msg - -proc subscribeSlotFreed(sales: Sales) {.async.} = - let context = sales.context - let market = context.market - - proc onSlotFreed(requestId: RequestId, slotIndex: uint64) = - sales.onSlotFreed(requestId, slotIndex) - - try: - let sub = await market.subscribeSlotFreed(onSlotFreed) - sales.subscriptions.add(sub) - except CancelledError as error: - raise error - except CatchableError as e: - error "Unable to subscribe to slot freed events", msg = e.msg - -proc subscribeSlotReservationsFull(sales: Sales) {.async.} = - let context = sales.context - let market = context.market - let queue = context.slotQueue - - proc onSlotReservationsFull(requestId: RequestId, slotIndex: uint64) = - if slotIndex > uint16.high.uint64: - error "Cannot cast slot index to uint16, value = ", slotIndex - return - - trace "reservations for slot full, removing from slot queue", requestId, slotIndex - queue.delete(requestId, slotIndex.uint16) - - try: - let sub = await market.subscribeSlotReservationsFull(onSlotReservationsFull) - sales.subscriptions.add(sub) - except CancelledError as error: - raise error - except CatchableError as e: - error "Unable to subscribe to slot filled events", msg = e.msg - -proc startSlotQueue(sales: Sales) = - let slotQueue = sales.context.slotQueue - let reservations = sales.context.reservations - - slotQueue.onProcessSlot = proc(item: SlotQueueItem) {.async: (raises: []).} = - trace "processing slot queue item", reqId = item.requestId, slotIdx = item.slotIndex - try: - await sales.processSlot(item) - except CancelledError: - discard - - slotQueue.start() - - proc OnAvailabilitySaved(availability: Availability) {.async: (raises: []).} = - if availability.enabled: - await sales.OnAvailabilitySaved(availability) - - reservations.OnAvailabilitySaved = OnAvailabilitySaved - -proc subscribe(sales: Sales) {.async.} = - await sales.subscribeRequested() - await sales.subscribeFulfilled() - await sales.subscribeFailure() - await sales.subscribeSlotFilled() - await sales.subscribeSlotFreed() - await sales.subscribeCancellation() - await sales.subscribeSlotReservationsFull() - -proc unsubscribe(sales: Sales) {.async.} = - for sub in sales.subscriptions: - try: - await sub.unsubscribe() - except CancelledError as error: - raise error - except CatchableError as e: - error "Unable to unsubscribe from subscription", error = e.msg - -proc start*(sales: Sales) {.async.} = - await sales.load() - sales.startSlotQueue() - await sales.subscribe() - sales.running = true - -proc stop*(sales: Sales) {.async.} = - trace "stopping sales" - sales.running = false - await sales.context.slotQueue.stop() - await sales.unsubscribe() - await sales.trackedFutures.cancelTracked() - - for agent in sales.agents: - await agent.stop() - - sales.agents = @[] diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim deleted file mode 100644 index 2e62fdf8..00000000 --- a/codex/sales/reservations.nim +++ /dev/null @@ -1,759 +0,0 @@ -## Logos Storage -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. -## -## +--------------------------------------+ -## | RESERVATION | -## +---------------------------------------------------+ |--------------------------------------| -## | AVAILABILITY | | ReservationId | id | PK | -## |---------------------------------------------------| |--------------------------------------| -## | AvailabilityId | id | PK |<-||-------o<-| AvailabilityId | availabilityId | FK | -## |---------------------------------------------------| |--------------------------------------| -## | UInt256 | totalSize | | | UInt256 | size | | -## |---------------------------------------------------| |--------------------------------------| -## | UInt256 | freeSize | | | UInt256 | slotIndex | | -## |---------------------------------------------------| +--------------------------------------+ -## | UInt256 | duration | | -## |---------------------------------------------------| -## | UInt256 | minPricePerBytePerSecond | | -## |---------------------------------------------------| -## | UInt256 | totalCollateral | | -## |---------------------------------------------------| -## | UInt256 | totalRemainingCollateral | | -## +---------------------------------------------------+ - -{.push raises: [], gcsafe.} - -import std/sequtils -import std/sugar -import std/typetraits -import std/sequtils -import std/times -import pkg/chronos -import pkg/datastore -import pkg/questionable -import pkg/questionable/results -import pkg/stint -import pkg/stew/byteutils -import ../codextypes -import ../logutils -import ../clock -import ../stores -import ../market -import ../contracts/requests -import ../utils/json -import ../units - -export requests -export logutils - -from nimcrypto import randomBytes - -logScope: - topics = "marketplace sales reservations" - -type - AvailabilityId* = distinct array[32, byte] - ReservationId* = distinct array[32, byte] - SomeStorableObject = Availability | Reservation - SomeStorableId = AvailabilityId | ReservationId - Availability* = ref object - id* {.serialize.}: AvailabilityId - totalSize* {.serialize.}: uint64 - freeSize* {.serialize.}: uint64 - duration* {.serialize.}: uint64 - minPricePerBytePerSecond* {.serialize.}: UInt256 - totalCollateral {.serialize.}: UInt256 - totalRemainingCollateral* {.serialize.}: UInt256 - # If set to false, the availability will not accept new slots. - # If enabled, it will not impact any existing slots that are already being hosted. - enabled* {.serialize.}: bool - # Specifies the latest timestamp after which the availability will no longer host any slots. - # If set to 0, there will be no restrictions. - until* {.serialize.}: SecondsSince1970 - - Reservation* = ref object - id* {.serialize.}: ReservationId - availabilityId* {.serialize.}: AvailabilityId - size* {.serialize.}: uint64 - requestId* {.serialize.}: RequestId - slotIndex* {.serialize.}: uint64 - validUntil* {.serialize.}: SecondsSince1970 - - Reservations* = ref object of RootObj - availabilityLock: AsyncLock - # Lock for protecting assertions of availability's sizes when searching for matching availability - repo: RepoStore - OnAvailabilitySaved: ?OnAvailabilitySaved - - GetNext* = proc(): Future[?seq[byte]] {.async: (raises: [CancelledError]), closure.} - IterDispose* = proc(): Future[?!void] {.async: (raises: [CancelledError]), closure.} - OnAvailabilitySaved* = - proc(availability: Availability): Future[void] {.async: (raises: []).} - StorableIter* = ref object - finished*: bool - next*: GetNext - dispose*: IterDispose - - ReservationsError* = object of CodexError - ReserveFailedError* = object of ReservationsError - ReleaseFailedError* = object of ReservationsError - DeleteFailedError* = object of ReservationsError - GetFailedError* = object of ReservationsError - NotExistsError* = object of ReservationsError - SerializationError* = object of ReservationsError - UpdateFailedError* = object of ReservationsError - BytesOutOfBoundsError* = object of ReservationsError - UntilOutOfBoundsError* = object of ReservationsError - -const - SalesKey = (CodexMetaKey / "sales").tryGet # TODO: move to sales module - ReservationsKey = (SalesKey / "reservations").tryGet - -proc hash*(x: AvailabilityId): Hash {.borrow.} -proc all*( - self: Reservations, T: type SomeStorableObject -): Future[?!seq[T]] {.async: (raises: [CancelledError]).} - -proc all*( - self: Reservations, T: type SomeStorableObject, availabilityId: AvailabilityId -): Future[?!seq[T]] {.async: (raises: [CancelledError]).} - -template withLock(lock, body) = - try: - await lock.acquire() - body - finally: - if lock.locked: - lock.release() - -proc new*(T: type Reservations, repo: RepoStore): Reservations = - T(availabilityLock: newAsyncLock(), repo: repo) - -proc init*( - _: type Availability, - totalSize: uint64, - freeSize: uint64, - duration: uint64, - minPricePerBytePerSecond: UInt256, - totalCollateral: UInt256, - enabled: bool, - until: SecondsSince1970, -): Availability = - var id: array[32, byte] - doAssert randomBytes(id) == 32 - Availability( - id: AvailabilityId(id), - totalSize: totalSize, - freeSize: freeSize, - duration: duration, - minPricePerBytePerSecond: minPricePerBytePerSecond, - totalCollateral: totalCollateral, - totalRemainingCollateral: totalCollateral, - enabled: enabled, - until: until, - ) - -func totalCollateral*(self: Availability): UInt256 {.inline.} = - return self.totalCollateral - -proc `totalCollateral=`*(self: Availability, value: UInt256) {.inline.} = - self.totalCollateral = value - self.totalRemainingCollateral = value - -proc init*( - _: type Reservation, - availabilityId: AvailabilityId, - size: uint64, - requestId: RequestId, - slotIndex: uint64, - validUntil: SecondsSince1970, -): Reservation = - var id: array[32, byte] - doAssert randomBytes(id) == 32 - Reservation( - id: ReservationId(id), - availabilityId: availabilityId, - size: size, - requestId: requestId, - slotIndex: slotIndex, - validUntil: validUntil, - ) - -func toArray(id: SomeStorableId): array[32, byte] = - array[32, byte](id) - -proc `==`*(x, y: AvailabilityId): bool {.borrow.} -proc `==`*(x, y: ReservationId): bool {.borrow.} -proc `==`*(x, y: Reservation): bool = - x.id == y.id - -proc `==`*(x, y: Availability): bool = - x.id == y.id - -proc `$`*(id: SomeStorableId): string = - id.toArray.toHex - -proc toErr[E1: ref CatchableError, E2: ReservationsError]( - e1: E1, _: type E2, msg: string = e1.msg -): ref E2 = - return newException(E2, msg, e1) - -logutils.formatIt(LogFormat.textLines, SomeStorableId): - it.short0xHexLog -logutils.formatIt(LogFormat.json, SomeStorableId): - it.to0xHexLog - -proc `OnAvailabilitySaved=`*( - self: Reservations, OnAvailabilitySaved: OnAvailabilitySaved -) = - self.OnAvailabilitySaved = some OnAvailabilitySaved - -func key*(id: AvailabilityId): ?!Key = - ## sales / reservations / - (ReservationsKey / $id) - -func key*(reservationId: ReservationId, availabilityId: AvailabilityId): ?!Key = - ## sales / reservations / / - (availabilityId.key / $reservationId) - -func key*(availability: Availability): ?!Key = - return availability.id.key - -func maxCollateralPerByte*(availability: Availability): UInt256 = - # If freeSize happens to be zero, we convention that the maxCollateralPerByte - # should be equal to totalRemainingCollateral. - if availability.freeSize == 0.uint64: - return availability.totalRemainingCollateral - - return availability.totalRemainingCollateral div availability.freeSize.stuint(256) - -func key*(reservation: Reservation): ?!Key = - return key(reservation.id, reservation.availabilityId) - -func available*(self: Reservations): uint = - self.repo.available.uint - -func hasAvailable*(self: Reservations, bytes: uint): bool = - self.repo.available(bytes.NBytes) - -proc exists*( - self: Reservations, key: Key -): Future[bool] {.async: (raises: [CancelledError]).} = - let exists = await self.repo.metaDs.ds.contains(key) - return exists - -iterator items(self: StorableIter): auto = - while not self.finished: - yield self.next() - -proc getImpl( - self: Reservations, key: Key -): Future[?!seq[byte]] {.async: (raises: [CancelledError]).} = - if not await self.exists(key): - let err = - newException(NotExistsError, "object with key " & $key & " does not exist") - return failure(err) - - without serialized =? await self.repo.metaDs.ds.get(key), error: - return failure(error.toErr(GetFailedError)) - - return success serialized - -proc get*( - self: Reservations, key: Key, T: type SomeStorableObject -): Future[?!T] {.async: (raises: [CancelledError]).} = - without serialized =? await self.getImpl(key), error: - return failure(error) - - without obj =? T.fromJson(serialized), error: - return failure(error.toErr(SerializationError)) - - return success obj - -proc updateImpl( - self: Reservations, obj: SomeStorableObject -): Future[?!void] {.async: (raises: [CancelledError]).} = - trace "updating " & $(obj.type), id = obj.id - - without key =? obj.key, error: - return failure(error) - - if err =? (await self.repo.metaDs.ds.put(key, @(obj.toJson.toBytes))).errorOption: - return failure(err.toErr(UpdateFailedError)) - - return success() - -proc updateAvailability( - self: Reservations, obj: Availability -): Future[?!void] {.async: (raises: [CancelledError]).} = - logScope: - availabilityId = obj.id - - if obj.until < 0: - let error = - newException(UntilOutOfBoundsError, "Cannot set until to a negative value") - return failure(error) - - without key =? obj.key, error: - return failure(error) - - without oldAvailability =? await self.get(key, Availability), err: - if err of NotExistsError: - trace "Creating new Availability" - let res = await self.updateImpl(obj) - # inform subscribers that Availability has been added - if OnAvailabilitySaved =? self.OnAvailabilitySaved: - await OnAvailabilitySaved(obj) - return res - else: - return failure(err) - - if obj.until > 0: - without allReservations =? await self.all(Reservation, obj.id), error: - error.msg = "Error updating reservation: " & error.msg - return failure(error) - - let requestEnds = allReservations.mapIt(it.validUntil) - - if requestEnds.len > 0 and requestEnds.max > obj.until: - let error = newException( - UntilOutOfBoundsError, - "Until parameter must be greater or equal to the longest currently hosted slot", - ) - return failure(error) - - # Sizing of the availability changed, we need to adjust the repo reservation accordingly - if oldAvailability.totalSize != obj.totalSize: - trace "totalSize changed, updating repo reservation" - if oldAvailability.totalSize < obj.totalSize: # storage added - if reserveErr =? ( - await self.repo.reserve((obj.totalSize - oldAvailability.totalSize).NBytes) - ).errorOption: - return failure(reserveErr.toErr(ReserveFailedError)) - elif oldAvailability.totalSize > obj.totalSize: # storage removed - if reserveErr =? ( - await self.repo.release((oldAvailability.totalSize - obj.totalSize).NBytes) - ).errorOption: - return failure(reserveErr.toErr(ReleaseFailedError)) - - let res = await self.updateImpl(obj) - - if oldAvailability.freeSize < obj.freeSize or oldAvailability.duration < obj.duration or - oldAvailability.minPricePerBytePerSecond < obj.minPricePerBytePerSecond or - oldAvailability.totalRemainingCollateral < obj.totalRemainingCollateral: - # availability updated - # inform subscribers that Availability has been modified (with increased - # size) - if OnAvailabilitySaved =? self.OnAvailabilitySaved: - await OnAvailabilitySaved(obj) - return res - -proc update*( - self: Reservations, obj: Reservation -): Future[?!void] {.async: (raises: [CancelledError]).} = - return await self.updateImpl(obj) - -proc update*( - self: Reservations, obj: Availability -): Future[?!void] {.async: (raises: [CancelledError]).} = - try: - withLock(self.availabilityLock): - return await self.updateAvailability(obj) - except AsyncLockError as e: - error "Lock error when trying to update the availability", err = e.msg - return failure(e) - -proc delete( - self: Reservations, key: Key -): Future[?!void] {.async: (raises: [CancelledError]).} = - trace "deleting object", key - - if not await self.exists(key): - return success() - - if err =? (await self.repo.metaDs.ds.delete(key)).errorOption: - return failure(err.toErr(DeleteFailedError)) - - return success() - -proc deleteReservation*( - self: Reservations, - reservationId: ReservationId, - availabilityId: AvailabilityId, - returnedCollateral: ?UInt256 = UInt256.none, -): Future[?!void] {.async: (raises: [CancelledError]).} = - logScope: - reservationId - availabilityId - - trace "deleting reservation" - - without key =? key(reservationId, availabilityId), error: - return failure(error) - - try: - withLock(self.availabilityLock): - without reservation =? (await self.get(key, Reservation)), error: - if error of NotExistsError: - return success() - else: - return failure(error) - - without availabilityKey =? availabilityId.key, error: - return failure(error) - - without var availability =? await self.get(availabilityKey, Availability), error: - return failure(error) - - if reservation.size > 0.uint64: - trace "returning remaining reservation bytes to availability", - size = reservation.size - availability.freeSize += reservation.size - - if collateral =? returnedCollateral: - availability.totalRemainingCollateral += collateral - - if updateErr =? (await self.updateAvailability(availability)).errorOption: - return failure(updateErr) - - if err =? (await self.repo.metaDs.ds.delete(key)).errorOption: - return failure(err.toErr(DeleteFailedError)) - - return success() - except AsyncLockError as e: - error "Lock error when trying to delete the availability", err = e.msg - return failure(e) - -# TODO: add support for deleting availabilities -# To delete, must not have any active sales. - -proc createAvailability*( - self: Reservations, - size: uint64, - duration: uint64, - minPricePerBytePerSecond: UInt256, - totalCollateral: UInt256, - enabled: bool, - until: SecondsSince1970, -): Future[?!Availability] {.async: (raises: [CancelledError]).} = - trace "creating availability", - size, duration, minPricePerBytePerSecond, totalCollateral, enabled, until - - if until < 0: - let error = - newException(UntilOutOfBoundsError, "Cannot set until to a negative value") - return failure(error) - - let availability = Availability.init( - size, size, duration, minPricePerBytePerSecond, totalCollateral, enabled, until - ) - let bytes = availability.freeSize - - if reserveErr =? (await self.repo.reserve(bytes.NBytes)).errorOption: - return failure(reserveErr.toErr(ReserveFailedError)) - - if updateErr =? (await self.update(availability)).errorOption: - # rollback the reserve - trace "rolling back reserve" - if rollbackErr =? (await self.repo.release(bytes.NBytes)).errorOption: - rollbackErr.parent = updateErr - return failure(rollbackErr) - - return failure(updateErr) - - return success(availability) - -method createReservation*( - self: Reservations, - availabilityId: AvailabilityId, - slotSize: uint64, - requestId: RequestId, - slotIndex: uint64, - collateralPerByte: UInt256, - validUntil: SecondsSince1970, -): Future[?!Reservation] {.async: (raises: [CancelledError]), base.} = - try: - withLock(self.availabilityLock): - without availabilityKey =? availabilityId.key, error: - return failure(error) - - without availability =? await self.get(availabilityKey, Availability), error: - return failure(error) - - # Check that the found availability has enough free space after the lock has been acquired, to prevent asynchronous Availiability modifications - if availability.freeSize < slotSize: - let error = newException( - BytesOutOfBoundsError, - "trying to reserve an amount of bytes that is greater than the free size of the Availability", - ) - return failure(error) - - trace "Creating reservation", - availabilityId, slotSize, requestId, slotIndex, validUntil = validUntil - - let reservation = - Reservation.init(availabilityId, slotSize, requestId, slotIndex, validUntil) - - if createResErr =? (await self.update(reservation)).errorOption: - return failure(createResErr) - - # reduce availability freeSize by the slot size, which is now accounted for in - # the newly created Reservation - availability.freeSize -= slotSize - - # adjust the remaining totalRemainingCollateral - availability.totalRemainingCollateral -= slotSize.u256 * collateralPerByte - - # update availability with reduced size - trace "Updating availability with reduced size", freeSize = availability.freeSize - if updateErr =? (await self.updateAvailability(availability)).errorOption: - trace "Updating availability failed, rolling back reservation creation" - - without key =? reservation.key, keyError: - keyError.parent = updateErr - return failure(keyError) - - # rollback the reservation creation - if rollbackErr =? (await self.delete(key)).errorOption: - rollbackErr.parent = updateErr - return failure(rollbackErr) - - return failure(updateErr) - - trace "Reservation succesfully created" - return success(reservation) - except AsyncLockError as e: - error "Lock error when trying to delete the availability", err = e.msg - return failure(e) - -proc returnBytesToAvailability*( - self: Reservations, - availabilityId: AvailabilityId, - reservationId: ReservationId, - bytes: uint64, -): Future[?!void] {.async: (raises: [CancelledError]).} = - logScope: - reservationId - availabilityId - try: - withLock(self.availabilityLock): - without key =? key(reservationId, availabilityId), error: - return failure(error) - - without var reservation =? (await self.get(key, Reservation)), error: - return failure(error) - - # We are ignoring bytes that are still present in the Reservation because - # they will be returned to Availability through `deleteReservation`. - let bytesToBeReturned = bytes - reservation.size - - if bytesToBeReturned == 0: - trace "No bytes are returned", - requestSizeBytes = bytes, returningBytes = bytesToBeReturned - return success() - - trace "Returning bytes", - requestSizeBytes = bytes, returningBytes = bytesToBeReturned - - # First lets see if we can re-reserve the bytes, if the Repo's quota - # is depleted then we will fail-fast as there is nothing to be done atm. - if reserveErr =? (await self.repo.reserve(bytesToBeReturned.NBytes)).errorOption: - return failure(reserveErr.toErr(ReserveFailedError)) - - without availabilityKey =? availabilityId.key, error: - return failure(error) - - without var availability =? await self.get(availabilityKey, Availability), error: - return failure(error) - - availability.freeSize += bytesToBeReturned - - # Update availability with returned size - if updateErr =? (await self.updateAvailability(availability)).errorOption: - trace "Rolling back returning bytes" - if rollbackErr =? (await self.repo.release(bytesToBeReturned.NBytes)).errorOption: - rollbackErr.parent = updateErr - return failure(rollbackErr) - - return failure(updateErr) - - return success() - except AsyncLockError as e: - error "Lock error when returning bytes to the availability", err = e.msg - return failure(e) - -proc release*( - self: Reservations, - reservationId: ReservationId, - availabilityId: AvailabilityId, - bytes: uint, -): Future[?!void] {.async: (raises: [CancelledError]).} = - logScope: - topics = "release" - bytes - reservationId - availabilityId - - trace "releasing bytes and updating reservation" - - without key =? key(reservationId, availabilityId), error: - return failure(error) - - without var reservation =? (await self.get(key, Reservation)), error: - return failure(error) - - if reservation.size < bytes: - let error = newException( - BytesOutOfBoundsError, - "trying to release an amount of bytes that is greater than the total size of the Reservation", - ) - return failure(error) - - if releaseErr =? (await self.repo.release(bytes.NBytes)).errorOption: - return failure(releaseErr.toErr(ReleaseFailedError)) - - reservation.size -= bytes - - # persist partially used Reservation with updated size - if err =? (await self.update(reservation)).errorOption: - # rollback release if an update error encountered - trace "rolling back release" - if rollbackErr =? (await self.repo.reserve(bytes.NBytes)).errorOption: - rollbackErr.parent = err - return failure(rollbackErr) - return failure(err) - - return success() - -proc storables( - self: Reservations, T: type SomeStorableObject, queryKey: Key = ReservationsKey -): Future[?!StorableIter] {.async: (raises: [CancelledError]).} = - var iter = StorableIter() - let query = Query.init(queryKey) - when T is Availability: - # should indicate key length of 4, but let the .key logic determine it - without defaultKey =? AvailabilityId.default.key, error: - return failure(error) - elif T is Reservation: - # should indicate key length of 5, but let the .key logic determine it - without defaultKey =? key(ReservationId.default, AvailabilityId.default), error: - return failure(error) - else: - raiseAssert "unknown type" - - without results =? await self.repo.metaDs.ds.query(query), error: - return failure(error) - - # /sales/reservations - proc next(): Future[?seq[byte]] {.async: (raises: [CancelledError]).} = - await idleAsync() - iter.finished = results.finished - if not results.finished and res =? (await results.next()) and res.data.len > 0 and - key =? res.key and key.namespaces.len == defaultKey.namespaces.len: - return some res.data - - return none seq[byte] - - proc dispose(): Future[?!void] {.async: (raises: [CancelledError]).} = - return await results.dispose() - - iter.next = next - iter.dispose = dispose - return success iter - -proc allImpl( - self: Reservations, T: type SomeStorableObject, queryKey: Key = ReservationsKey -): Future[?!seq[T]] {.async: (raises: [CancelledError]).} = - var ret: seq[T] = @[] - - without storables =? (await self.storables(T, queryKey)), error: - return failure(error) - - for storable in storables.items: - try: - without bytes =? (await storable): - continue - - without obj =? T.fromJson(bytes), error: - error "json deserialization error", - json = string.fromBytes(bytes), error = error.msg - continue - - ret.add obj - except CancelledError as err: - raise err - except CatchableError as err: - error "Error when retrieving storable", error = err.msg - continue - - return success(ret) - -proc all*( - self: Reservations, T: type SomeStorableObject -): Future[?!seq[T]] {.async: (raises: [CancelledError]).} = - return await self.allImpl(T) - -proc all*( - self: Reservations, T: type SomeStorableObject, availabilityId: AvailabilityId -): Future[?!seq[T]] {.async: (raises: [CancelledError]).} = - without key =? key(availabilityId): - return failure("no key") - - return await self.allImpl(T, key) - -proc findAvailability*( - self: Reservations, - size, duration: uint64, - pricePerBytePerSecond, collateralPerByte: UInt256, - validUntil: SecondsSince1970, -): Future[?Availability] {.async: (raises: [CancelledError]).} = - without storables =? (await self.storables(Availability)), e: - error "failed to get all storables", error = e.msg - return none Availability - - for item in storables.items: - if bytes =? (await item) and availability =? Availability.fromJson(bytes): - if availability.enabled and size <= availability.freeSize and - duration <= availability.duration and - collateralPerByte <= availability.maxCollateralPerByte and - pricePerBytePerSecond >= availability.minPricePerBytePerSecond and - (availability.until == 0 or availability.until >= validUntil): - trace "availability matched", - id = availability.id, - enabled = availability.enabled, - size, - availFreeSize = availability.freeSize, - duration, - availDuration = availability.duration, - pricePerBytePerSecond, - availMinPricePerBytePerSecond = availability.minPricePerBytePerSecond, - collateralPerByte, - availMaxCollateralPerByte = availability.maxCollateralPerByte, - until = availability.until - - # TODO: As soon as we're on ARC-ORC, we can use destructors - # to automatically dispose our iterators when they fall out of scope. - # For now: - if err =? (await storables.dispose()).errorOption: - error "failed to dispose storables iter", error = err.msg - return none Availability - return some availability - - trace "availability did not match", - id = availability.id, - enabled = availability.enabled, - size, - availFreeSize = availability.freeSize, - duration, - availDuration = availability.duration, - pricePerBytePerSecond, - availMinPricePerBytePerSecond = availability.minPricePerBytePerSecond, - collateralPerByte, - availMaxCollateralPerByte = availability.maxCollateralPerByte, - until = availability.until diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim deleted file mode 100644 index 6584353c..00000000 --- a/codex/sales/salesagent.nim +++ /dev/null @@ -1,152 +0,0 @@ -import pkg/chronos -import pkg/questionable -import pkg/questionable/results -import pkg/stint -import ../contracts/requests -import ../errors -import ../logutils -import ../utils/exceptions -import ./statemachine -import ./salescontext -import ./salesdata -import ./reservations -import ./slotqueue - -export reservations - -logScope: - topics = "marketplace sales" - -type - SalesAgent* = ref object of Machine - context*: SalesContext - data*: SalesData - subscribed: bool - # Slot-level callbacks. - onCleanUp*: OnCleanUp - onFilled*: ?OnFilled - - OnCleanUp* = proc(reprocessSlot = false, returnedCollateral = UInt256.none) {. - async: (raises: []) - .} - OnFilled* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, raises: [].} - - SalesAgentError = object of CodexError - AllSlotsFilledError* = object of SalesAgentError - -func `==`*(a, b: SalesAgent): bool = - a.data.requestId == b.data.requestId and a.data.slotIndex == b.data.slotIndex - -proc newSalesAgent*( - context: SalesContext, - requestId: RequestId, - slotIndex: uint64, - request: ?StorageRequest, - slotQueueItem = SlotQueueItem.none, -): SalesAgent = - var agent = SalesAgent.new() - agent.context = context - agent.data = SalesData( - requestId: requestId, - slotIndex: slotIndex, - request: request, - slotQueueItem: slotQueueItem, - ) - return agent - -proc retrieveRequest*(agent: SalesAgent) {.async.} = - let data = agent.data - let market = agent.context.market - if data.request.isNone: - data.request = await market.getRequest(data.requestId) - -proc retrieveRequestState*(agent: SalesAgent): Future[?RequestState] {.async.} = - let data = agent.data - let market = agent.context.market - return await market.requestState(data.requestId) - -func state*(agent: SalesAgent): ?string = - proc description(state: State): string = - $state - - agent.query(description) - -proc subscribeCancellation(agent: SalesAgent) {.async.} = - let data = agent.data - let clock = agent.context.clock - - proc onCancelled() {.async: (raises: []).} = - without request =? data.request: - return - - try: - let market = agent.context.market - let expiry = await market.requestExpiresAt(data.requestId) - - while true: - let deadline = max(clock.now, expiry) + 1 - trace "Waiting for request to be cancelled", now = clock.now, expiry = deadline - await clock.waitUntil(deadline) - - without state =? await agent.retrieveRequestState(): - error "Unknown request", requestId = data.requestId - return - - case state - of New: - discard - of RequestState.Cancelled: - agent.schedule(cancelledEvent(request)) - break - of RequestState.Started, RequestState.Finished, RequestState.Failed: - break - - debug "The request is not yet canceled, even though it should be. Waiting for some more time.", - currentState = state, now = clock.now - except CancelledError: - trace "Waiting for expiry to lapse was cancelled", requestId = data.requestId - except CatchableError as e: - error "Error while waiting for expiry to lapse", error = e.msgDetail - - data.cancelled = onCancelled() - -method onFulfilled*( - agent: SalesAgent, requestId: RequestId -) {.base, gcsafe, raises: [].} = - let cancelled = agent.data.cancelled - if agent.data.requestId == requestId and not cancelled.isNil and not cancelled.finished: - cancelled.cancelSoon() - -method onFailed*(agent: SalesAgent, requestId: RequestId) {.base, gcsafe, raises: [].} = - without request =? agent.data.request: - return - if agent.data.requestId == requestId: - agent.schedule(failedEvent(request)) - -method onSlotFilled*( - agent: SalesAgent, requestId: RequestId, slotIndex: uint64 -) {.base, gcsafe, raises: [].} = - if agent.data.requestId == requestId and agent.data.slotIndex == slotIndex: - agent.schedule(slotFilledEvent(requestId, slotIndex)) - -proc subscribe*(agent: SalesAgent) {.async.} = - if agent.subscribed: - return - - await agent.subscribeCancellation() - agent.subscribed = true - -proc unsubscribe*(agent: SalesAgent) {.async: (raises: []).} = - if not agent.subscribed: - return - - let data = agent.data - if not data.cancelled.isNil and not data.cancelled.finished: - await data.cancelled.cancelAndWait() - data.cancelled = nil - - agent.subscribed = false - -proc stop*(agent: SalesAgent) {.async: (raises: []).} = - await Machine(agent).stop() - await agent.unsubscribe() diff --git a/codex/sales/salescontext.nim b/codex/sales/salescontext.nim deleted file mode 100644 index 5fd7099c..00000000 --- a/codex/sales/salescontext.nim +++ /dev/null @@ -1,42 +0,0 @@ -import pkg/questionable -import pkg/questionable/results -import pkg/libp2p/cid - -import ../market -import ../clock -import ./slotqueue -import ./reservations -import ../blocktype as bt - -type - SalesContext* = ref object - market*: Market - clock*: Clock - # Sales-level callbacks. Closure will be overwritten each time a slot is - # processed. - onStore*: ?OnStore - onClear*: ?OnClear - onSale*: ?OnSale - onProve*: ?OnProve - onExpiryUpdate*: ?OnExpiryUpdate - reservations*: Reservations - slotQueue*: SlotQueue - simulateProofFailures*: int - - BlocksCb* = - proc(blocks: seq[bt.Block]): Future[?!void] {.async: (raises: [CancelledError]).} - OnStore* = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - blocksCb: BlocksCb, - isRepairing: bool, - ): Future[?!void] {.async: (raises: [CancelledError]).} - OnProve* = proc(slot: Slot, challenge: ProofChallenge): Future[?!Groth16Proof] {. - async: (raises: [CancelledError]) - .} - OnExpiryUpdate* = proc(rootCid: Cid, expiry: SecondsSince1970): Future[?!void] {. - async: (raises: [CancelledError]) - .} - OnClear* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, raises: [].} - OnSale* = proc(request: StorageRequest, slotIndex: uint64) {.gcsafe, raises: [].} diff --git a/codex/sales/salesdata.nim b/codex/sales/salesdata.nim deleted file mode 100644 index ec16fef1..00000000 --- a/codex/sales/salesdata.nim +++ /dev/null @@ -1,14 +0,0 @@ -import pkg/chronos -import ../contracts/requests -import ../market -import ./reservations -import ./slotqueue - -type SalesData* = ref object - requestId*: RequestId - ask*: StorageAsk - request*: ?StorageRequest - slotIndex*: uint64 - cancelled*: Future[void] - reservation*: ?Reservation - slotQueueItem*: ?SlotQueueItem diff --git a/codex/sales/slotqueue.nim b/codex/sales/slotqueue.nim deleted file mode 100644 index ad9a07db..00000000 --- a/codex/sales/slotqueue.nim +++ /dev/null @@ -1,408 +0,0 @@ -import std/sequtils -import std/tables -import pkg/chronos -import pkg/questionable -import pkg/questionable/results -import ../errors -import ../logutils -import ../rng -import ../utils -import ../contracts/requests -import ../utils/asyncheapqueue -import ../utils/trackedfutures - -logScope: - topics = "marketplace slotqueue" - -type - OnProcessSlot* = proc(item: SlotQueueItem): Future[void] {.async: (raises: []).} - - # Non-ref obj copies value when assigned, preventing accidental modification - # of values which could cause an incorrect order (eg - # ``slotQueue[1].collateral = 1`` would cause ``collateral`` to be updated, - # but the heap invariant would no longer be honoured. When non-ref, the - # compiler can ensure that statement will fail). - SlotQueueItem* = object - requestId: RequestId - slotIndex: uint16 - slotSize: uint64 - duration: uint64 - pricePerBytePerSecond: UInt256 - collateral: UInt256 # Collateral computed - expiry: ?uint64 - seen: bool - - # don't need to -1 to prevent overflow when adding 1 (to always allow push) - # because AsyncHeapQueue size is of type `int`, which is larger than `uint16` - SlotQueueSize = range[1'u16 .. uint16.high] - - SlotQueue* = ref object - maxWorkers: int - onProcessSlot: ?OnProcessSlot - queue: AsyncHeapQueue[SlotQueueItem] - running: bool - trackedFutures: TrackedFutures - unpaused: AsyncEvent - - SlotQueueError = object of CodexError - SlotQueueItemExistsError* = object of SlotQueueError - SlotQueueItemNotExistsError* = object of SlotQueueError - SlotsOutOfRangeError* = object of SlotQueueError - QueueNotRunningError* = object of SlotQueueError - -# Number of concurrent workers used for processing SlotQueueItems -const DefaultMaxWorkers = 3 - -# Cap slot queue size to prevent unbounded growth and make sifting more -# efficient. Max size is not equivalent to the number of slots a host can -# service, which is limited by host availabilities and new requests circulating -# the network. Additionally, each new request/slot in the network will be -# included in the queue if it is higher priority than any of the exisiting -# items. Older slots should be unfillable over time as other hosts fill the -# slots. -const DefaultMaxSize = 128'u16 - -proc profitability(item: SlotQueueItem): UInt256 = - StorageAsk( - duration: item.duration, - pricePerBytePerSecond: item.pricePerBytePerSecond, - slotSize: item.slotSize, - ).pricePerSlot - -proc `<`*(a, b: SlotQueueItem): bool = - # for A to have a higher priority than B (in a min queue), A must be less than - # B. - var scoreA: uint8 = 0 - var scoreB: uint8 = 0 - - proc addIf(score: var uint8, condition: bool, addition: int) = - if condition: - score += 1'u8 shl addition - - scoreA.addIf(a.seen < b.seen, 4) - scoreB.addIf(a.seen > b.seen, 4) - - scoreA.addIf(a.profitability > b.profitability, 3) - scoreB.addIf(a.profitability < b.profitability, 3) - - scoreA.addIf(a.collateral < b.collateral, 2) - scoreB.addIf(a.collateral > b.collateral, 2) - - if expiryA =? a.expiry and expiryB =? b.expiry: - scoreA.addIf(expiryA > expiryB, 1) - scoreB.addIf(expiryA < expiryB, 1) - - return scoreA > scoreB - -proc `==`*(a, b: SlotQueueItem): bool = - a.requestId == b.requestId and a.slotIndex == b.slotIndex - -proc new*( - _: type SlotQueue, - maxWorkers = DefaultMaxWorkers, - maxSize: SlotQueueSize = DefaultMaxSize, -): SlotQueue = - if maxWorkers <= 0: - raise newException(ValueError, "maxWorkers must be positive") - if maxWorkers.uint16 > maxSize: - raise newException(ValueError, "maxWorkers must be less than maxSize") - - SlotQueue( - maxWorkers: maxWorkers, - # Add 1 to always allow for an extra item to be pushed onto the queue - # temporarily. After push (and sort), the bottom-most item will be deleted - queue: newAsyncHeapQueue[SlotQueueItem](maxSize.int + 1), - running: false, - trackedFutures: TrackedFutures.new(), - unpaused: newAsyncEvent(), - ) - # avoid instantiating `workers` in constructor to avoid side effects in - # `newAsyncQueue` procedure - -proc init*( - _: type SlotQueueItem, - requestId: RequestId, - slotIndex: uint16, - ask: StorageAsk, - expiry: ?uint64, - collateral: UInt256, - seen = false, -): SlotQueueItem = - SlotQueueItem( - requestId: requestId, - slotIndex: slotIndex, - slotSize: ask.slotSize, - duration: ask.duration, - pricePerBytePerSecond: ask.pricePerBytePerSecond, - collateral: collateral, - expiry: expiry, - seen: seen, - ) - -proc init*( - _: type SlotQueueItem, - requestId: RequestId, - slotIndex: uint16, - ask: StorageAsk, - expiry: uint64, - collateral: UInt256, - seen = false, -): SlotQueueItem = - SlotQueueItem.init(requestId, slotIndex, ask, some expiry, collateral, seen) - -proc init*( - _: type SlotQueueItem, - request: StorageRequest, - slotIndex: uint16, - collateral: UInt256, -): SlotQueueItem = - SlotQueueItem.init(request.id, slotIndex, request.ask, request.expiry, collateral) - -proc init*( - _: type SlotQueueItem, - requestId: RequestId, - ask: StorageAsk, - expiry: ?uint64, - collateral: UInt256, -): seq[SlotQueueItem] {.raises: [SlotsOutOfRangeError].} = - if not ask.slots.inRange: - raise newException(SlotsOutOfRangeError, "Too many slots") - - var i = 0'u16 - proc initSlotQueueItem(): SlotQueueItem = - let item = SlotQueueItem.init(requestId, i, ask, expiry, collateral) - inc i - return item - - var items = newSeqWith(ask.slots.int, initSlotQueueItem()) - Rng.instance.shuffle(items) - return items - -proc init*( - _: type SlotQueueItem, - requestId: RequestId, - ask: StorageAsk, - expiry: uint64, - collateral: UInt256, -): seq[SlotQueueItem] {.raises: [SlotsOutOfRangeError].} = - SlotQueueItem.init(requestId, ask, some expiry, collateral) - -proc init*( - _: type SlotQueueItem, request: StorageRequest, collateral: UInt256 -): seq[SlotQueueItem] = - return SlotQueueItem.init(request.id, request.ask, uint64.none, collateral) - -proc inRange*(val: SomeUnsignedInt): bool = - val.uint16 in SlotQueueSize.low .. SlotQueueSize.high - -proc requestId*(self: SlotQueueItem): RequestId = - self.requestId - -proc slotIndex*(self: SlotQueueItem): uint16 = - self.slotIndex - -proc slotSize*(self: SlotQueueItem): uint64 = - self.slotSize - -proc duration*(self: SlotQueueItem): uint64 = - self.duration - -proc pricePerBytePerSecond*(self: SlotQueueItem): UInt256 = - self.pricePerBytePerSecond - -proc collateralPerByte*(self: SlotQueueItem): UInt256 = - self.collateralPerByte - -proc seen*(self: SlotQueueItem): bool = - self.seen - -proc `seen=`*(self: var SlotQueueItem, seen: bool) = - self.seen = seen - -proc running*(self: SlotQueue): bool = - self.running - -proc len*(self: SlotQueue): int = - self.queue.len - -proc size*(self: SlotQueue): int = - self.queue.size - 1 - -proc paused*(self: SlotQueue): bool = - not self.unpaused.isSet - -proc `$`*(self: SlotQueue): string = - $self.queue - -proc `onProcessSlot=`*(self: SlotQueue, onProcessSlot: OnProcessSlot) = - self.onProcessSlot = some onProcessSlot - -proc contains*(self: SlotQueue, item: SlotQueueItem): bool = - self.queue.contains(item) - -proc pause*(self: SlotQueue) = - # set unpaused flag to false -- coroutines will block on unpaused.wait() - self.unpaused.clear() - -proc unpause*(self: SlotQueue) = - # set unpaused flag to true -- unblocks coroutines waiting on unpaused.wait() - self.unpaused.fire() - -proc push*(self: SlotQueue, item: SlotQueueItem): ?!void {.raises: [].} = - logScope: - requestId = item.requestId - slotIndex = item.slotIndex - seen = item.seen - - trace "pushing item to queue" - - if not self.running: - let err = newException(QueueNotRunningError, "queue not running") - return failure(err) - - if self.contains(item): - let err = newException(SlotQueueItemExistsError, "item already exists") - return failure(err) - - if err =? self.queue.pushNoWait(item).mapFailure.errorOption: - return failure(err) - - if self.queue.full(): - # delete the last item - self.queue.del(self.queue.size - 1) - - doAssert self.queue.len <= self.queue.size - 1 - - # when slots are pushed to the queue, the queue should be unpaused if it was - # paused - if self.paused and not item.seen: - trace "unpausing queue after new slot pushed" - self.unpause() - - return success() - -proc push*(self: SlotQueue, items: seq[SlotQueueItem]): ?!void = - for item in items: - if err =? self.push(item).errorOption: - return failure(err) - - return success() - -proc findByRequest(self: SlotQueue, requestId: RequestId): seq[SlotQueueItem] = - var items: seq[SlotQueueItem] = @[] - for item in self.queue.items: - if item.requestId == requestId: - items.add item - return items - -proc delete*(self: SlotQueue, item: SlotQueueItem) = - logScope: - requestId = item.requestId - slotIndex = item.slotIndex - - trace "removing item from queue" - - if not self.running: - trace "cannot delete item from queue, queue not running" - return - - self.queue.delete(item) - -proc delete*(self: SlotQueue, requestId: RequestId, slotIndex: uint16) = - let item = SlotQueueItem(requestId: requestId, slotIndex: slotIndex) - self.delete(item) - -proc delete*(self: SlotQueue, requestId: RequestId) = - let items = self.findByRequest(requestId) - for item in items: - self.delete(item) - -proc `[]`*(self: SlotQueue, i: Natural): SlotQueueItem = - self.queue[i] - -proc clearSeenFlags*(self: SlotQueue) = - # Enumerate all items in the queue, overwriting each item with `seen = false`. - # To avoid issues with new queue items being pushed to the queue while all - # items are being iterated (eg if a new storage request comes in and pushes - # new slots to the queue), this routine must remain synchronous. - - if self.queue.empty: - return - - for item in self.queue.mitems: - item.seen = false # does not maintain the heap invariant - - # force heap reshuffling to maintain the heap invariant - doAssert self.queue.update(self.queue[0]), "slot queue failed to reshuffle" - - trace "all 'seen' flags cleared" - -proc runWorker(self: SlotQueue) {.async: (raises: []).} = - trace "slot queue worker loop started" - while self.running: - try: - if self.paused: - trace "Queue is paused, waiting for new slots or availabilities to be modified/added" - - # block until unpaused is true/fired, ie wait for queue to be unpaused - await self.unpaused.wait() - - let item = await self.queue.pop() # if queue empty, wait here for new items - - logScope: - reqId = item.requestId - slotIdx = item.slotIndex - seen = item.seen - - if not self.running: # may have changed after waiting for pop - trace "not running, exiting" - break - - # If, upon processing a slot, the slot item already has a `seen` flag set, - # the queue should be paused. - if item.seen: - trace "processing already seen item, pausing queue", - reqId = item.requestId, slotIdx = item.slotIndex - self.pause() - # put item back in queue so that if other items are pushed while paused, - # it will be sorted accordingly. Otherwise, this item would be processed - # immediately (with priority over other items) once unpaused - trace "readding seen item back into the queue" - discard self.push(item) # on error, drop the item and continue - continue - - trace "processing item" - without onProcessSlot =? self.onProcessSlot: - raiseAssert "slot queue onProcessSlot not set" - - await onProcessSlot(item) - except CancelledError: - trace "slot queue worker cancelled" - break - except CatchableError as e: # raised from self.queue.pop() - warn "slot queue worker error encountered during processing", error = e.msg - trace "slot queue worker loop stopped" - -proc start*(self: SlotQueue) = - if self.running: - return - - trace "starting slot queue" - - self.running = true - - # Add initial workers to the `AsyncHeapQueue`. Once a worker has completed its - # task, a new worker will be pushed to the queue - for i in 0 ..< self.maxWorkers: - let worker = self.runWorker() - self.trackedFutures.track(worker) - -proc stop*(self: SlotQueue) {.async.} = - if not self.running: - return - - trace "stopping slot queue" - - self.running = false - - await self.trackedFutures.cancelTracked() diff --git a/codex/sales/statemachine.nim b/codex/sales/statemachine.nim deleted file mode 100644 index dc199ade..00000000 --- a/codex/sales/statemachine.nim +++ /dev/null @@ -1,41 +0,0 @@ -import pkg/questionable -import ../errors -import ../utils/asyncstatemachine -import ../market -import ../clock -import ../contracts/requests - -export market -export clock -export asyncstatemachine - -type - SaleState* = ref object of State - SaleError* = object of CodexError - -method onCancelled*( - state: SaleState, request: StorageRequest -): ?State {.base, raises: [].} = - discard - -method onFailed*( - state: SaleState, request: StorageRequest -): ?State {.base, raises: [].} = - discard - -method onSlotFilled*( - state: SaleState, requestId: RequestId, slotIndex: uint64 -): ?State {.base, raises: [].} = - discard - -proc cancelledEvent*(request: StorageRequest): Event = - return proc(state: State): ?State = - SaleState(state).onCancelled(request) - -proc failedEvent*(request: StorageRequest): Event = - return proc(state: State): ?State = - SaleState(state).onFailed(request) - -proc slotFilledEvent*(requestId: RequestId, slotIndex: uint64): Event = - return proc(state: State): ?State = - SaleState(state).onSlotFilled(requestId, slotIndex) diff --git a/codex/sales/states/cancelled.nim b/codex/sales/states/cancelled.nim deleted file mode 100644 index f3c755a3..00000000 --- a/codex/sales/states/cancelled.nim +++ /dev/null @@ -1,62 +0,0 @@ -import ../../logutils -import ../../utils/exceptions -import ../salesagent -import ../statemachine -import ./errored - -logScope: - topics = "marketplace sales cancelled" - -type SaleCancelled* = ref object of SaleState - -method `$`*(state: SaleCancelled): string = - "SaleCancelled" - -proc slotIsFilledByMe( - market: Market, requestId: RequestId, slotIndex: uint64 -): Future[bool] {.async: (raises: [CancelledError, MarketError]).} = - let host = await market.getHost(requestId, slotIndex) - let me = await market.getSigner() - - return host == me.some - -method run*( - state: SaleCancelled, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - let market = agent.context.market - - without request =? data.request: - raiseAssert "no sale request" - - try: - var returnedCollateral = UInt256.none - - if await slotIsFilledByMe(market, data.requestId, data.slotIndex): - debug "Collecting collateral and partial payout", - requestId = data.requestId, slotIndex = data.slotIndex - - let slot = Slot(request: request, slotIndex: data.slotIndex) - let currentCollateral = await market.currentCollateral(slot.id) - - try: - await market.freeSlot(slot.id) - except SlotStateMismatchError as e: - warn "Failed to free slot because slot is already free", error = e.msg - - returnedCollateral = currentCollateral.some - - if onClear =? agent.context.onClear and request =? data.request: - onClear(request, data.slotIndex) - - if onCleanUp =? agent.onCleanUp: - await onCleanUp(reprocessSlot = false, returnedCollateral = returnedCollateral) - - warn "Sale cancelled due to timeout", - requestId = data.requestId, slotIndex = data.slotIndex - except CancelledError as e: - trace "SaleCancelled.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleCancelled.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/downloading.nim b/codex/sales/states/downloading.nim deleted file mode 100644 index 0d628962..00000000 --- a/codex/sales/states/downloading.nim +++ /dev/null @@ -1,96 +0,0 @@ -import pkg/questionable -import pkg/questionable/results - -import ../../blocktype as bt -import ../../logutils -import ../../market -import ../../utils/exceptions -import ../salesagent -import ../statemachine -import ./cancelled -import ./failed -import ./filled -import ./initialproving -import ./errored - -type SaleDownloading* = ref object of SaleState - -logScope: - topics = "marketplace sales downloading" - -method `$`*(state: SaleDownloading): string = - "SaleDownloading" - -method onCancelled*(state: SaleDownloading, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SaleDownloading, request: StorageRequest): ?State = - return some State(SaleFailed()) - -method onSlotFilled*( - state: SaleDownloading, requestId: RequestId, slotIndex: uint64 -): ?State = - return some State(SaleFilled()) - -method run*( - state: SaleDownloading, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - let context = agent.context - let market = context.market - let reservations = context.reservations - - without onStore =? context.onStore: - raiseAssert "onStore callback not set" - - without request =? data.request: - raiseAssert "no sale request" - - without reservation =? data.reservation: - raiseAssert("no reservation") - - logScope: - requestId = request.id - slotIndex = data.slotIndex - reservationId = reservation.id - availabilityId = reservation.availabilityId - - proc onBlocks( - blocks: seq[bt.Block] - ): Future[?!void] {.async: (raises: [CancelledError]).} = - # release batches of blocks as they are written to disk and - # update availability size - var bytes: uint = 0 - for blk in blocks: - if not blk.cid.isEmpty: - bytes += blk.data.len.uint - - trace "Releasing batch of bytes written to disk", bytes - return await reservations.release(reservation.id, reservation.availabilityId, bytes) - - try: - let requestId = request.id - let slotId = slotId(requestId, data.slotIndex) - let requestState = await market.requestState(requestId) - let isRepairing = (await market.slotState(slotId)) == SlotState.Repair - - trace "Retrieving expiry" - var expiry: SecondsSince1970 - if state =? requestState and state == RequestState.Started: - expiry = await market.getRequestEnd(requestId) - else: - expiry = await market.requestExpiresAt(requestId) - - trace "Starting download" - if err =? - (await onStore(request, expiry, data.slotIndex, onBlocks, isRepairing)).errorOption: - return some State(SaleErrored(error: err, reprocessSlot: false)) - - trace "Download complete" - return some State(SaleInitialProving()) - except CancelledError as e: - trace "SaleDownloading.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleDownloading.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/errored.nim b/codex/sales/states/errored.nim deleted file mode 100644 index 3887c652..00000000 --- a/codex/sales/states/errored.nim +++ /dev/null @@ -1,40 +0,0 @@ -import pkg/questionable -import pkg/questionable/results - -import ../statemachine -import ../salesagent -import ../../logutils -import ../../utils/exceptions - -logScope: - topics = "marketplace sales errored" - -type SaleErrored* = ref object of SaleState - error*: ref CatchableError - reprocessSlot*: bool - -method `$`*(state: SaleErrored): string = - "SaleErrored" - -method run*( - state: SaleErrored, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - let context = agent.context - - error "Sale error", - error = state.error.msgDetail, - requestId = data.requestId, - slotIndex = data.slotIndex - - try: - if onClear =? context.onClear and request =? data.request: - onClear(request, data.slotIndex) - - if onCleanUp =? agent.onCleanUp: - await onCleanUp(reprocessSlot = state.reprocessSlot) - except CancelledError as e: - trace "SaleErrored.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleErrored.run", error = e.msgDetail diff --git a/codex/sales/states/failed.nim b/codex/sales/states/failed.nim deleted file mode 100644 index f1490d20..00000000 --- a/codex/sales/states/failed.nim +++ /dev/null @@ -1,40 +0,0 @@ -import ../../logutils -import ../../utils/exceptions -import ../../utils/exceptions -import ../salesagent -import ../statemachine -import ./errored - -logScope: - topics = "marketplace sales failed" - -type - SaleFailed* = ref object of SaleState - SaleFailedError* = object of SaleError - -method `$`*(state: SaleFailed): string = - "SaleFailed" - -method run*( - state: SaleFailed, machine: Machine -): Future[?State] {.async: (raises: []).} = - let data = SalesAgent(machine).data - let market = SalesAgent(machine).context.market - - without request =? data.request: - raiseAssert "no sale request" - - try: - let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Removing slot from mySlots", - requestId = data.requestId, slotIndex = data.slotIndex - - await market.freeSlot(slot.id) - - let error = newException(SaleFailedError, "Sale failed") - return some State(SaleErrored(error: error)) - except CancelledError as e: - trace "SaleFailed.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleFailed.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/filled.nim b/codex/sales/states/filled.nim deleted file mode 100644 index ec54762a..00000000 --- a/codex/sales/states/filled.nim +++ /dev/null @@ -1,77 +0,0 @@ -import pkg/questionable -import pkg/questionable/results - -import ../../conf -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ../salesagent -import ./errored -import ./cancelled -import ./failed -import ./proving - -when storage_enable_proof_failures: - import ./provingsimulated - -logScope: - topics = "marketplace sales filled" - -type - SaleFilled* = ref object of SaleState - HostMismatchError* = object of CatchableError - -method onCancelled*(state: SaleFilled, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SaleFilled, request: StorageRequest): ?State = - return some State(SaleFailed()) - -method `$`*(state: SaleFilled): string = - "SaleFilled" - -method run*( - state: SaleFilled, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - let context = agent.context - let market = context.market - - try: - let host = await market.getHost(data.requestId, data.slotIndex) - let me = await market.getSigner() - - if host == me.some: - info "Slot succesfully filled", - requestId = data.requestId, slotIndex = data.slotIndex - - without request =? data.request: - raiseAssert "no sale request" - - if onFilled =? agent.onFilled: - onFilled(request, data.slotIndex) - - without onExpiryUpdate =? context.onExpiryUpdate: - raiseAssert "onExpiryUpdate callback not set" - - let requestEnd = await market.getRequestEnd(data.requestId) - if err =? (await onExpiryUpdate(request.content.cid, requestEnd)).errorOption: - return some State(SaleErrored(error: err)) - - when storage_enable_proof_failures: - if context.simulateProofFailures > 0: - info "Proving with failure rate", rate = context.simulateProofFailures - return some State( - SaleProvingSimulated(failEveryNProofs: context.simulateProofFailures) - ) - - return some State(SaleProving()) - else: - let error = newException(HostMismatchError, "Slot filled by other host") - return some State(SaleErrored(error: error)) - except CancelledError as e: - trace "SaleFilled.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleFilled.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim deleted file mode 100644 index f0fcd4f3..00000000 --- a/codex/sales/states/filling.nim +++ /dev/null @@ -1,63 +0,0 @@ -import pkg/stint -import ../../logutils -import ../../market -import ../../utils/exceptions -import ../statemachine -import ../salesagent -import ./filled -import ./cancelled -import ./failed -import ./ignored -import ./errored - -logScope: - topics = "marketplace sales filling" - -type SaleFilling* = ref object of SaleState - proof*: Groth16Proof - -method `$`*(state: SaleFilling): string = - "SaleFilling" - -method onCancelled*(state: SaleFilling, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SaleFilling, request: StorageRequest): ?State = - return some State(SaleFailed()) - -method run*( - state: SaleFilling, machine: Machine -): Future[?State] {.async: (raises: []).} = - let data = SalesAgent(machine).data - let market = SalesAgent(machine).context.market - - without (request =? data.request): - raiseAssert "Request not set" - - logScope: - requestId = data.requestId - slotIndex = data.slotIndex - - try: - without collateral =? await market.slotCollateral(data.requestId, data.slotIndex), - err: - error "Failure attempting to fill slot: unable to calculate collateral", - error = err.msg - return some State(SaleErrored(error: err)) - - debug "Filling slot" - try: - await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) - except SlotStateMismatchError as e: - debug "Slot is already filled, ignoring slot" - return some State(SaleIgnored(reprocessSlot: false, returnsCollateral: true)) - except MarketError as e: - return some State(SaleErrored(error: e)) - # other CatchableErrors are handled "automatically" by the SaleState - - return some State(SaleFilled()) - except CancelledError as e: - trace "SaleFilling.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleFilling.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/finished.nim b/codex/sales/states/finished.nim deleted file mode 100644 index 16e66d27..00000000 --- a/codex/sales/states/finished.nim +++ /dev/null @@ -1,48 +0,0 @@ -import pkg/chronos - -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ../salesagent -import ./cancelled -import ./failed -import ./errored - -logScope: - topics = "marketplace sales finished" - -type SaleFinished* = ref object of SaleState - returnedCollateral*: ?UInt256 - -method `$`*(state: SaleFinished): string = - "SaleFinished" - -method onCancelled*(state: SaleFinished, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SaleFinished, request: StorageRequest): ?State = - return some State(SaleFailed()) - -method run*( - state: SaleFinished, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - - without request =? data.request: - raiseAssert "no sale request" - - info "Slot finished and paid out", - requestId = data.requestId, slotIndex = data.slotIndex - - try: - if onClear =? agent.context.onClear: - onClear(request, data.slotIndex) - - if onCleanUp =? agent.onCleanUp: - await onCleanUp(returnedCollateral = state.returnedCollateral) - except CancelledError as e: - trace "SaleFilled.run onCleanUp was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleFilled.run in onCleanUp callback", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/ignored.nim b/codex/sales/states/ignored.nim deleted file mode 100644 index ca0d48f7..00000000 --- a/codex/sales/states/ignored.nim +++ /dev/null @@ -1,51 +0,0 @@ -import pkg/chronos - -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ../salesagent -import ./errored - -logScope: - topics = "marketplace sales ignored" - -# Ignored slots could mean there was no availability or that the slot could -# not be reserved. - -type SaleIgnored* = ref object of SaleState - reprocessSlot*: bool # readd slot to queue with `seen` flag - returnsCollateral*: bool # returns collateral when a reservation was created - -method `$`*(state: SaleIgnored): string = - "SaleIgnored" - -method run*( - state: SaleIgnored, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - let market = agent.context.market - - without request =? data.request: - raiseAssert "no sale request" - - var returnedCollateral = UInt256.none - - try: - if state.returnsCollateral: - # The returnedCollateral is needed because a reservation could - # be created and the collateral assigned to that reservation. - # The returnedCollateral will be used in the cleanup function - # and be passed to the deleteReservation function. - let slot = Slot(request: request, slotIndex: data.slotIndex) - returnedCollateral = request.ask.collateralPerSlot.some - - if onCleanUp =? agent.onCleanUp: - await onCleanUp( - reprocessSlot = state.reprocessSlot, returnedCollateral = returnedCollateral - ) - except CancelledError as e: - trace "SaleIgnored.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleIgnored.run in onCleanUp", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/initialproving.nim b/codex/sales/states/initialproving.nim deleted file mode 100644 index 57e8cc2c..00000000 --- a/codex/sales/states/initialproving.nim +++ /dev/null @@ -1,71 +0,0 @@ -import pkg/questionable/results -import ../../clock -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ../salesagent -import ./filling -import ./cancelled -import ./errored -import ./failed - -logScope: - topics = "marketplace sales initial-proving" - -type SaleInitialProving* = ref object of SaleState - -method `$`*(state: SaleInitialProving): string = - "SaleInitialProving" - -method onCancelled*(state: SaleInitialProving, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SaleInitialProving, request: StorageRequest): ?State = - return some State(SaleFailed()) - -proc waitUntilNextPeriod(clock: Clock, periodicity: Periodicity) {.async.} = - trace "Waiting until next period" - let period = periodicity.periodOf(clock.now().Timestamp) - let periodEnd = periodicity.periodEnd(period) - await clock.waitUntil((periodEnd + 1).toSecondsSince1970) - -proc waitForStableChallenge(market: Market, clock: Clock, slotId: SlotId) {.async.} = - let periodicity = await market.periodicity() - let downtime = await market.proofDowntime() - await clock.waitUntilNextPeriod(periodicity) - while (await market.getPointer(slotId)) > (256 - downtime): - await clock.waitUntilNextPeriod(periodicity) - -method run*( - state: SaleInitialProving, machine: Machine -): Future[?State] {.async: (raises: []).} = - let data = SalesAgent(machine).data - let context = SalesAgent(machine).context - let market = context.market - let clock = context.clock - - without request =? data.request: - raiseAssert "no sale request" - - without onProve =? context.onProve: - raiseAssert "onProve callback not set" - - try: - debug "Waiting for a proof challenge that is valid for the entire period" - let slot = Slot(request: request, slotIndex: data.slotIndex) - await waitForStableChallenge(market, clock, slot.id) - - debug "Generating initial proof", requestId = data.requestId - let challenge = await context.market.getChallenge(slot.id) - without proof =? (await onProve(slot, challenge)), err: - error "Failed to generate initial proof", error = err.msg - return some State(SaleErrored(error: err)) - - debug "Finished proof calculation", requestId = data.requestId - - return some State(SaleFilling(proof: proof)) - except CancelledError as e: - trace "SaleInitialProving.run onCleanUp was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleInitialProving.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/payout.nim b/codex/sales/states/payout.nim deleted file mode 100644 index e808307d..00000000 --- a/codex/sales/states/payout.nim +++ /dev/null @@ -1,46 +0,0 @@ -import ../../logutils -import ../../market -import ../../utils/exceptions -import ../statemachine -import ../salesagent -import ./cancelled -import ./failed -import ./finished -import ./errored - -logScope: - topics = "marketplace sales payout" - -type SalePayout* = ref object of SaleState - -method `$`*(state: SalePayout): string = - "SalePayout" - -method onCancelled*(state: SalePayout, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SalePayout, request: StorageRequest): ?State = - return some State(SaleFailed()) - -method run*( - state: SalePayout, machine: Machine -): Future[?State] {.async: (raises: []).} = - let data = SalesAgent(machine).data - let market = SalesAgent(machine).context.market - - without request =? data.request: - raiseAssert "no sale request" - - try: - let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Collecting finished slot's reward", - requestId = data.requestId, slotIndex = data.slotIndex - let currentCollateral = await market.currentCollateral(slot.id) - await market.freeSlot(slot.id) - - return some State(SaleFinished(returnedCollateral: some currentCollateral)) - except CancelledError as e: - trace "SalePayout.run onCleanUp was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SalePayout.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim deleted file mode 100644 index ff6eb88c..00000000 --- a/codex/sales/states/preparing.nim +++ /dev/null @@ -1,110 +0,0 @@ -import pkg/questionable -import pkg/questionable/results -import pkg/metrics - -import ../../logutils -import ../../market -import ../../utils/exceptions -import ../salesagent -import ../statemachine -import ./cancelled -import ./failed -import ./filled -import ./ignored -import ./slotreserving -import ./errored - -declareCounter( - codex_reservations_availability_mismatch, "codex reservations availability_mismatch" -) - -type SalePreparing* = ref object of SaleState - -logScope: - topics = "marketplace sales preparing" - -method `$`*(state: SalePreparing): string = - "SalePreparing" - -method onCancelled*(state: SalePreparing, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SalePreparing, request: StorageRequest): ?State = - return some State(SaleFailed()) - -method onSlotFilled*( - state: SalePreparing, requestId: RequestId, slotIndex: uint64 -): ?State = - return some State(SaleFilled()) - -method run*( - state: SalePreparing, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - let context = agent.context - let market = context.market - let reservations = context.reservations - - try: - await agent.retrieveRequest() - await agent.subscribe() - - without request =? data.request: - error "request could not be retrieved", id = data.requestId - let error = newException(SaleError, "request could not be retrieved") - return some State(SaleErrored(error: error)) - - let slotId = slotId(data.requestId, data.slotIndex) - let state = await market.slotState(slotId) - if state != SlotState.Free and state != SlotState.Repair: - return some State(SaleIgnored(reprocessSlot: false)) - - # TODO: Once implemented, check to ensure the host is allowed to fill the slot, - # due to the [sliding window mechanism](https://github.com/logos-storage/logos-storage-research/blob/master/design/marketplace.md#dispersal) - - logScope: - slotIndex = data.slotIndex - slotSize = request.ask.slotSize - duration = request.ask.duration - pricePerBytePerSecond = request.ask.pricePerBytePerSecond - collateralPerByte = request.ask.collateralPerByte - - let requestEnd = await market.getRequestEnd(data.requestId) - - without availability =? - await reservations.findAvailability( - request.ask.slotSize, request.ask.duration, request.ask.pricePerBytePerSecond, - request.ask.collateralPerByte, requestEnd, - ): - debug "No availability found for request, ignoring" - - return some State(SaleIgnored(reprocessSlot: true)) - - info "Availability found for request, creating reservation" - - without reservation =? - await noCancel reservations.createReservation( - availability.id, request.ask.slotSize, request.id, data.slotIndex, - request.ask.collateralPerByte, requestEnd, - ), error: - trace "Creation of reservation failed" - # Race condition: - # reservations.findAvailability (line 64) is no guarantee. You can never know for certain that the reservation can be created until after you have it. - # Should createReservation fail because there's no space, we proceed to SaleIgnored. - if error of BytesOutOfBoundsError: - # Lets monitor how often this happen and if it is often we can make it more inteligent to handle it - codex_reservations_availability_mismatch.inc() - return some State(SaleIgnored(reprocessSlot: true)) - - return some State(SaleErrored(error: error)) - - trace "Reservation created successfully" - - data.reservation = some reservation - return some State(SaleSlotReserving()) - except CancelledError as e: - trace "SalePreparing.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SalePreparing.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/proving.nim b/codex/sales/states/proving.nim deleted file mode 100644 index 690e9136..00000000 --- a/codex/sales/states/proving.nim +++ /dev/null @@ -1,166 +0,0 @@ -import std/options -import pkg/questionable/results -import ../../clock -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ../salesagent -import ../salescontext -import ./cancelled -import ./failed -import ./errored -import ./payout - -logScope: - topics = "marketplace sales proving" - -type - SlotFreedError* = object of CatchableError - SlotNotFilledError* = object of CatchableError - SaleProving* = ref object of SaleState - loop: Future[void] - -method prove*( - state: SaleProving, - slot: Slot, - challenge: ProofChallenge, - onProve: OnProve, - market: Market, - currentPeriod: Period, -) {.base, async.} = - try: - without proof =? (await onProve(slot, challenge)), err: - error "Failed to generate proof", error = err.msg - # In this state, there's nothing we can do except try again next time. - return - debug "Submitting proof", currentPeriod = currentPeriod, slotId = slot.id - await market.submitProof(slot.id, proof) - except CancelledError as error: - trace "Submitting proof cancelled" - raise error - except CatchableError as e: - error "Submitting proof failed", msg = e.msgDetail - -proc proveLoop( - state: SaleProving, - market: Market, - clock: Clock, - request: StorageRequest, - slotIndex: uint64, - onProve: OnProve, -) {.async.} = - let slot = Slot(request: request, slotIndex: slotIndex) - let slotId = slot.id - - logScope: - period = currentPeriod - requestId = request.id - slotIndex - slotId = slot.id - - proc getCurrentPeriod(): Future[Period] {.async.} = - let periodicity = await market.periodicity() - return periodicity.periodOf(clock.now().Timestamp) - - proc waitUntilPeriod(period: Period) {.async.} = - let periodicity = await market.periodicity() - # Ensure that we're past the period boundary by waiting an additional second - await clock.waitUntil((periodicity.periodStart(period) + 1).toSecondsSince1970) - - while true: - let currentPeriod = await getCurrentPeriod() - let slotState = await market.slotState(slot.id) - - case slotState - of SlotState.Filled: - debug "Proving for new period", period = currentPeriod - if (await market.isProofRequired(slotId)) or - (await market.willProofBeRequired(slotId)): - let challenge = await market.getChallenge(slotId) - debug "Proof is required", period = currentPeriod, challenge = challenge - await state.prove(slot, challenge, onProve, market, currentPeriod) - of SlotState.Cancelled: - debug "Slot reached cancelled state" - # do nothing, let onCancelled callback take care of it - of SlotState.Repair: - warn "Slot was forcible freed" - let message = "Slot was forcible freed and host was removed from its hosting" - raise newException(SlotFreedError, message) - of SlotState.Failed: - debug "Slot reached failed state" - # do nothing, let onFailed callback take care of it - of SlotState.Finished: - debug "Slot reached finished state", period = currentPeriod - return # exit the loop - else: - let message = "Slot is not in Filled state, but in state: " & $slotState - raise newException(SlotNotFilledError, message) - - debug "waiting until next period" - await waitUntilPeriod(currentPeriod + 1) - -method `$`*(state: SaleProving): string = - "SaleProving" - -method onCancelled*(state: SaleProving, request: StorageRequest): ?State = - # state.loop cancellation happens automatically when run is cancelled due to - # state change - return some State(SaleCancelled()) - -method onFailed*(state: SaleProving, request: StorageRequest): ?State = - # state.loop cancellation happens automatically when run is cancelled due to - # state change - return some State(SaleFailed()) - -method run*( - state: SaleProving, machine: Machine -): Future[?State] {.async: (raises: []).} = - let data = SalesAgent(machine).data - let context = SalesAgent(machine).context - - without request =? data.request: - raiseAssert "no sale request" - - without onProve =? context.onProve: - raiseAssert "onProve callback not set" - - without market =? context.market: - raiseAssert("market not set") - - without clock =? context.clock: - raiseAssert("clock not set") - - try: - debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex - try: - let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve) - state.loop = loop - await loop - except CancelledError as e: - trace "proving loop cancelled" - discard - except CatchableError as e: - error "Proving failed", - msg = e.msg, typ = $(type e), stack = e.getStackTrace(), error = e.msgDetail - return some State(SaleErrored(error: e)) - finally: - # Cleanup of the proving loop - debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex - - if not state.loop.isNil: - if not state.loop.finished: - try: - await state.loop.cancelAndWait() - except CancelledError: - discard - except CatchableError as e: - error "Error during cancellation of proving loop", msg = e.msg - - state.loop = nil - - return some State(SalePayout()) - except CancelledError as e: - trace "SaleProving.run onCleanUp was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleProving.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/provingsimulated.nim b/codex/sales/states/provingsimulated.nim deleted file mode 100644 index 8ba468e7..00000000 --- a/codex/sales/states/provingsimulated.nim +++ /dev/null @@ -1,57 +0,0 @@ -import ../../conf -when storage_enable_proof_failures: - import std/strutils - import pkg/stint - import pkg/ethers - - import ../../contracts/marketplace - import ../../contracts/requests - import ../../logutils - import ../../market - import ../../utils/exceptions - import ../salescontext - import ./proving - import ./errored - - logScope: - topics = "marketplace sales simulated-proving" - - type SaleProvingSimulated* = ref object of SaleProving - failEveryNProofs*: int - proofCount: int - - proc onSubmitProofError(error: ref CatchableError, period: Period, slotId: SlotId) = - error "Submitting invalid proof failed", period, slotId, msg = error.msgDetail - - method prove*( - state: SaleProvingSimulated, - slot: Slot, - challenge: ProofChallenge, - onProve: OnProve, - market: Market, - currentPeriod: Period, - ) {.async.} = - try: - trace "Processing proving in simulated mode" - state.proofCount += 1 - if state.failEveryNProofs > 0 and state.proofCount mod state.failEveryNProofs == 0: - state.proofCount = 0 - - try: - warn "Submitting INVALID proof", period = currentPeriod, slotId = slot.id - await market.submitProof(slot.id, Groth16Proof.default) - except ProofInvalidError as e: - discard # expected - except CancelledError as error: - raise error - except CatchableError as e: - onSubmitProofError(e, currentPeriod, slot.id) - else: - await procCall SaleProving(state).prove( - slot, challenge, onProve, market, currentPeriod - ) - except CancelledError as e: - trace "Submitting INVALID proof cancelled", error = e.msgDetail - raise e - except CatchableError as e: - error "Submitting INVALID proof failed", error = e.msgDetail diff --git a/codex/sales/states/slotreserving.nim b/codex/sales/states/slotreserving.nim deleted file mode 100644 index 4842f302..00000000 --- a/codex/sales/states/slotreserving.nim +++ /dev/null @@ -1,65 +0,0 @@ -import pkg/questionable -import pkg/metrics - -import ../../logutils -import ../../market -import ../../utils/exceptions -import ../salesagent -import ../statemachine -import ./cancelled -import ./failed -import ./ignored -import ./downloading -import ./errored - -type SaleSlotReserving* = ref object of SaleState - -logScope: - topics = "marketplace sales reserving" - -method `$`*(state: SaleSlotReserving): string = - "SaleSlotReserving" - -method onCancelled*(state: SaleSlotReserving, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SaleSlotReserving, request: StorageRequest): ?State = - return some State(SaleFailed()) - -method run*( - state: SaleSlotReserving, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - let context = agent.context - let market = context.market - - logScope: - requestId = data.requestId - slotIndex = data.slotIndex - - try: - let canReserve = await market.canReserveSlot(data.requestId, data.slotIndex) - if canReserve: - try: - trace "Reserving slot" - await market.reserveSlot(data.requestId, data.slotIndex) - except SlotReservationNotAllowedError as e: - debug "Slot cannot be reserved, ignoring", error = e.msg - return some State(SaleIgnored(reprocessSlot: false, returnsCollateral: true)) - except MarketError as e: - return some State(SaleErrored(error: e)) - # other CatchableErrors are handled "automatically" by the SaleState - - trace "Slot successfully reserved" - return some State(SaleDownloading()) - else: - # do not re-add this slot to the queue, and return bytes from Reservation to - # the Availability - debug "Slot cannot be reserved, ignoring" - return some State(SaleIgnored(reprocessSlot: false, returnsCollateral: true)) - except CancelledError as e: - trace "SaleSlotReserving.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleSlotReserving.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/sales/states/unknown.nim b/codex/sales/states/unknown.nim deleted file mode 100644 index b714a4b9..00000000 --- a/codex/sales/states/unknown.nim +++ /dev/null @@ -1,73 +0,0 @@ -import ../../logutils -import ../../utils/exceptions -import ../statemachine -import ../salesagent -import ./filled -import ./finished -import ./failed -import ./errored -import ./proving -import ./cancelled -import ./payout - -logScope: - topics = "marketplace sales unknown" - -type - SaleUnknown* = ref object of SaleState - SaleUnknownError* = object of CatchableError - UnexpectedSlotError* = object of SaleUnknownError - -method `$`*(state: SaleUnknown): string = - "SaleUnknown" - -method onCancelled*(state: SaleUnknown, request: StorageRequest): ?State = - return some State(SaleCancelled()) - -method onFailed*(state: SaleUnknown, request: StorageRequest): ?State = - return some State(SaleFailed()) - -method run*( - state: SaleUnknown, machine: Machine -): Future[?State] {.async: (raises: []).} = - let agent = SalesAgent(machine) - let data = agent.data - let market = agent.context.market - - try: - await agent.retrieveRequest() - await agent.subscribe() - - without request =? data.request: - error "request could not be retrieved", id = data.requestId - let error = newException(SaleError, "request could not be retrieved") - return some State(SaleErrored(error: error)) - - let slotId = slotId(data.requestId, data.slotIndex) - let slotState = await market.slotState(slotId) - - case slotState - of SlotState.Free: - let error = - newException(UnexpectedSlotError, "Slot state on chain should not be 'free'") - return some State(SaleErrored(error: error)) - of SlotState.Filled: - return some State(SaleFilled()) - of SlotState.Finished: - return some State(SalePayout()) - of SlotState.Paid: - return some State(SaleFinished()) - of SlotState.Failed: - return some State(SaleFailed()) - of SlotState.Cancelled: - return some State(SaleCancelled()) - of SlotState.Repair: - let error = newException( - SlotFreedError, "Slot was forcible freed and host was removed from its hosting" - ) - return some State(SaleErrored(error: error)) - except CancelledError as e: - trace "SaleUnknown.run was cancelled", error = e.msgDetail - except CatchableError as e: - error "Error during SaleUnknown.run", error = e.msgDetail - return some State(SaleErrored(error: e)) diff --git a/codex/slots.nim b/codex/slots.nim deleted file mode 100644 index 0fe9d59e..00000000 --- a/codex/slots.nim +++ /dev/null @@ -1,6 +0,0 @@ -import ./slots/builder -import ./slots/sampler -import ./slots/proofs -import ./slots/types - -export builder, sampler, proofs, types diff --git a/codex/slots/builder.nim b/codex/slots/builder.nim deleted file mode 100644 index 25844db6..00000000 --- a/codex/slots/builder.nim +++ /dev/null @@ -1,8 +0,0 @@ -import ./builder/builder -import ./converters - -import ../merkletree - -export builder, converters - -type Poseidon2Builder* = SlotsBuilder[Poseidon2Tree, Poseidon2Hash] diff --git a/codex/slots/builder/builder.nim b/codex/slots/builder/builder.nim deleted file mode 100644 index 633e19ea..00000000 --- a/codex/slots/builder/builder.nim +++ /dev/null @@ -1,404 +0,0 @@ -## Logos Storage -## Copyright (c) 2023 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [].} - -import std/math -import std/sequtils -import std/sugar - -import pkg/libp2p -import pkg/chronos -import pkg/questionable -import pkg/questionable/results -import pkg/constantine/math/io/io_fields - -import ../../logutils -import ../../utils -import ../../stores -import ../../manifest -import ../../merkletree -import ../../utils/asynciter -import ../../indexingstrategy - -import ../converters - -export converters, asynciter - -logScope: - topics = "codex slotsbuilder" - -type SlotsBuilder*[T, H] = ref object of RootObj - store: BlockStore - manifest: Manifest # current manifest - strategy: IndexingStrategy # indexing strategy - cellSize: NBytes # cell size - numSlotBlocks: Natural - # number of blocks per slot (should yield a power of two number of cells) - slotRoots: seq[H] # roots of the slots - emptyBlock: seq[byte] # empty block - verifiableTree: ?T # verification tree (dataset tree) - emptyDigestTree: T # empty digest tree for empty blocks - -func verifiable*[T, H](self: SlotsBuilder[T, H]): bool {.inline.} = - ## Returns true if the slots are verifiable. - ## - - self.manifest.verifiable - -func slotRoots*[T, H](self: SlotsBuilder[T, H]): seq[H] {.inline.} = - ## Returns the slot roots. - ## - - self.slotRoots - -func verifyTree*[T, H](self: SlotsBuilder[T, H]): ?T {.inline.} = - ## Returns the slots tree (verification tree). - ## - - self.verifiableTree - -func verifyRoot*[T, H](self: SlotsBuilder[T, H]): ?H {.inline.} = - ## Returns the slots root (verification root). - ## - - if tree =? self.verifyTree and root =? tree.root: - return some root - -func numSlots*[T, H](self: SlotsBuilder[T, H]): Natural = - ## Number of slots. - ## - - self.manifest.numSlots - -func numSlotBlocks*[T, H](self: SlotsBuilder[T, H]): Natural = - ## Number of blocks per slot. - ## - - self.numSlotBlocks - -func numBlocks*[T, H](self: SlotsBuilder[T, H]): Natural = - ## Number of blocks. - ## - - self.numSlotBlocks * self.manifest.numSlots - -func slotBytes*[T, H](self: SlotsBuilder[T, H]): NBytes = - ## Number of bytes per slot. - ## - - (self.manifest.blockSize.int * self.numSlotBlocks).NBytes - -func numBlockCells*[T, H](self: SlotsBuilder[T, H]): Natural = - ## Number of cells per block. - ## - - (self.manifest.blockSize div self.cellSize).Natural - -func cellSize*[T, H](self: SlotsBuilder[T, H]): NBytes = - ## Cell size. - ## - - self.cellSize - -func numSlotCells*[T, H](self: SlotsBuilder[T, H]): Natural = - ## Number of cells per slot. - ## - - self.numBlockCells * self.numSlotBlocks - -func slotIndicesIter*[T, H](self: SlotsBuilder[T, H], slot: Natural): ?!Iter[int] = - ## Returns the slot indices. - ## - - self.strategy.getIndices(slot).catch - -func slotIndices*[T, H](self: SlotsBuilder[T, H], slot: Natural): seq[int] = - ## Returns the slot indices. - ## - - if iter =? self.strategy.getIndices(slot).catch: - return toSeq(iter) - -func manifest*[T, H](self: SlotsBuilder[T, H]): Manifest = - ## Returns the manifest. - ## - - self.manifest - -proc buildBlockTree*[T, H]( - self: SlotsBuilder[T, H], blkIdx: Natural, slotPos: Natural -): Future[?!(seq[byte], T)] {.async: (raises: [CancelledError]).} = - ## Build the block digest tree and return a tuple with the - ## block data and the tree. - ## - - logScope: - blkIdx = blkIdx - slotPos = slotPos - numSlotBlocks = self.manifest.numSlotBlocks - cellSize = self.cellSize - - trace "Building block tree" - - if slotPos > (self.manifest.numSlotBlocks - 1): - # pad blocks are 0 byte blocks - trace "Returning empty digest tree for pad block" - return success (self.emptyBlock, self.emptyDigestTree) - - without blk =? await self.store.getBlock(self.manifest.treeCid, blkIdx), err: - error "Failed to get block CID for tree at index", err = err.msg - return failure(err) - - if blk.isEmpty: - success (self.emptyBlock, self.emptyDigestTree) - else: - without tree =? T.digestTree(blk.data, self.cellSize.int), err: - error "Failed to create digest for block", err = err.msg - return failure(err) - - success (blk.data, tree) - -proc getCellHashes*[T, H]( - self: SlotsBuilder[T, H], slotIndex: Natural -): Future[?!seq[H]] {.async: (raises: [CancelledError, IndexingError]).} = - ## Collect all the cells from a block and return - ## their hashes. - ## - - let - treeCid = self.manifest.treeCid - blockCount = self.manifest.blocksCount - numberOfSlots = self.manifest.numSlots - - logScope: - treeCid = treeCid - origBlockCount = blockCount - numberOfSlots = numberOfSlots - slotIndex = slotIndex - - let hashes = collect(newSeq): - for i, blkIdx in self.strategy.getIndices(slotIndex): - logScope: - blkIdx = blkIdx - pos = i - - trace "Getting block CID for tree at index" - without (_, tree) =? (await self.buildBlockTree(blkIdx, i)) and digest =? tree.root, - err: - error "Failed to get block CID for tree at index", err = err.msg - return failure(err) - - trace "Get block digest", digest = digest.toHex - digest - - success hashes - -proc buildSlotTree*[T, H]( - self: SlotsBuilder[T, H], slotIndex: Natural -): Future[?!T] {.async: (raises: [CancelledError]).} = - ## Build the slot tree from the block digest hashes - ## and return the tree. - - try: - without cellHashes =? (await self.getCellHashes(slotIndex)), err: - error "Failed to select slot blocks", err = err.msg - return failure(err) - - T.init(cellHashes) - except IndexingError as err: - error "Failed to build slot tree", err = err.msg - return failure(err) - -proc buildSlot*[T, H]( - self: SlotsBuilder[T, H], slotIndex: Natural -): Future[?!H] {.async: (raises: [CancelledError]).} = - ## Build a slot tree and store the proofs in - ## the block store. - ## - - logScope: - cid = self.manifest.treeCid - slotIndex = slotIndex - - trace "Building slot tree" - - without tree =? (await self.buildSlotTree(slotIndex)) and - treeCid =? tree.root .? toSlotCid, err: - error "Failed to build slot tree", err = err.msg - return failure(err) - - trace "Storing slot tree", treeCid, slotIndex, leaves = tree.leavesCount - for i, leaf in tree.leaves: - without cellCid =? leaf.toCellCid, err: - error "Failed to get CID for slot cell", err = err.msg - return failure(err) - - without proof =? tree.getProof(i) and encodableProof =? proof.toEncodableProof, err: - error "Failed to get proof for slot tree", err = err.msg - return failure(err) - - if err =? - (await self.store.putCidAndProof(treeCid, i, cellCid, encodableProof)).errorOption: - error "Failed to store slot tree", err = err.msg - return failure(err) - - tree.root() - -func buildVerifyTree*[T, H](self: SlotsBuilder[T, H], slotRoots: openArray[H]): ?!T = - T.init(@slotRoots) - -proc buildSlots*[T, H]( - self: SlotsBuilder[T, H] -): Future[?!void] {.async: (raises: [CancelledError]).} = - ## Build all slot trees and store them in the block store. - ## - - logScope: - cid = self.manifest.treeCid - blockCount = self.manifest.blocksCount - - trace "Building slots" - - if self.slotRoots.len == 0: - self.slotRoots = collect(newSeq): - for i in 0 ..< self.manifest.numSlots: - without slotRoot =? (await self.buildSlot(i)), err: - error "Failed to build slot", err = err.msg, index = i - return failure(err) - slotRoot - - without tree =? self.buildVerifyTree(self.slotRoots) and root =? tree.root, err: - error "Failed to build slot roots tree", err = err.msg - return failure(err) - - if verifyTree =? self.verifyTree and verifyRoot =? verifyTree.root: - if not bool(verifyRoot == root): # TODO: `!=` doesn't work for SecretBool - return failure "Existing slots root doesn't match reconstructed root." - - self.verifiableTree = some tree - - success() - -proc buildManifest*[T, H]( - self: SlotsBuilder[T, H] -): Future[?!Manifest] {.async: (raises: [CancelledError]).} = - if err =? (await self.buildSlots()).errorOption: - error "Failed to build slot roots", err = err.msg - return failure(err) - - without rootCids =? self.slotRoots.toSlotCids(), err: - error "Failed to map slot roots to CIDs", err = err.msg - return failure(err) - - without rootProvingCidRes =? self.verifyRoot .? toVerifyCid() and - rootProvingCid =? rootProvingCidRes, err: - error "Failed to map slot roots to CIDs", err = err.msg - return failure(err) - - Manifest.new( - self.manifest, rootProvingCid, rootCids, self.cellSize, self.strategy.strategyType - ) - -proc new*[T, H]( - _: type SlotsBuilder[T, H], - store: BlockStore, - manifest: Manifest, - strategy = LinearStrategy, - cellSize = DefaultCellSize, -): ?!SlotsBuilder[T, H] = - if not manifest.protected: - trace "Manifest is not protected." - return failure("Manifest is not protected.") - - logScope: - blockSize = manifest.blockSize - strategy = strategy - cellSize = cellSize - - if (manifest.blocksCount mod manifest.numSlots) != 0: - const msg = "Number of blocks must be divisible by number of slots." - trace msg - return failure(msg) - - let cellSize = if manifest.verifiable: manifest.cellSize else: cellSize - if (manifest.blockSize mod cellSize) != 0.NBytes: - const msg = "Block size must be divisible by cell size." - trace msg - return failure(msg) - - let - numSlotBlocks = manifest.numSlotBlocks - numBlockCells = (manifest.blockSize div cellSize).int # number of cells per block - numSlotCells = manifest.numSlotBlocks * numBlockCells - # number of uncorrected slot cells - pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot - numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks - # pow2 blocks per slot - - numSlotBlocksTotal = - # pad blocks per slot - if numPadSlotBlocks > 0: - numPadSlotBlocks + numSlotBlocks - else: - numSlotBlocks - - numBlocksTotal = numSlotBlocksTotal * manifest.numSlots # number of blocks per slot - - emptyBlock = newSeq[byte](manifest.blockSize.int) - emptyDigestTree = ?T.digestTree(emptyBlock, cellSize.int) - - strategy = - ?strategy.init( - 0, - manifest.blocksCount - 1, - manifest.numSlots, - manifest.numSlots, - numPadSlotBlocks, - ).catch - - logScope: - numSlotBlocks = numSlotBlocks - numBlockCells = numBlockCells - numSlotCells = numSlotCells - pow2SlotCells = pow2SlotCells - numPadSlotBlocks = numPadSlotBlocks - numBlocksTotal = numBlocksTotal - numSlotBlocksTotal = numSlotBlocksTotal - strategy = strategy.strategyType - - trace "Creating slots builder" - - var self = SlotsBuilder[T, H]( - store: store, - manifest: manifest, - strategy: strategy, - cellSize: cellSize, - emptyBlock: emptyBlock, - numSlotBlocks: numSlotBlocksTotal, - emptyDigestTree: emptyDigestTree, - ) - - if manifest.verifiable: - if manifest.slotRoots.len == 0 or manifest.slotRoots.len != manifest.numSlots: - return failure "Manifest is verifiable but slot roots are missing or invalid." - - let - slotRoots = manifest.slotRoots.mapIt((?it.fromSlotCid())) - tree = ?self.buildVerifyTree(slotRoots) - expectedRoot = ?manifest.verifyRoot.fromVerifyCid() - verifyRoot = ?tree.root - - if verifyRoot != expectedRoot: - return failure "Existing slots root doesn't match reconstructed root." - - self.slotRoots = slotRoots - self.verifiableTree = some tree - - success self diff --git a/codex/slots/converters.nim b/codex/slots/converters.nim deleted file mode 100644 index f22a7d64..00000000 --- a/codex/slots/converters.nim +++ /dev/null @@ -1,82 +0,0 @@ -## Logos Storage -## Copyright (c) 2024 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/sequtils - -import pkg/libp2p -import pkg/stew/arrayops -import pkg/questionable -import pkg/questionable/results -import pkg/poseidon2 -import pkg/poseidon2/io - -import ../codextypes -import ../merkletree -import ../errors -import ../utils/digest - -func toCid(hash: Poseidon2Hash, mcodec: MultiCodec, cidCodec: MultiCodec): ?!Cid = - let - mhash = ?MultiHash.init(mcodec, hash.toBytes).mapFailure - treeCid = ?Cid.init(CIDv1, cidCodec, mhash).mapFailure - success treeCid - -proc toPoseidon2Hash( - cid: Cid, mcodec: MultiCodec, cidCodec: MultiCodec -): ?!Poseidon2Hash = - if cid.cidver != CIDv1: - return failure("Unexpected CID version") - - if cid.mcodec != cidCodec: - return failure( - "Cid is not of expected codec. Was: " & $cid.mcodec & " but expected: " & $cidCodec - ) - - let - mhash = ?cid.mhash.mapFailure - bytes: array[32, byte] = array[32, byte].initCopyFrom(mhash.digestBytes()) - hash = ?Poseidon2Hash.fromBytes(bytes).toFailure - - success hash - -func toCellCid*(hash: Poseidon2Hash): ?!Cid = - toCid(hash, Pos2Bn128MrklCodec, CodexSlotCellCodec) - -func fromCellCid*(cid: Cid): ?!Poseidon2Hash = - toPoseidon2Hash(cid, Pos2Bn128MrklCodec, CodexSlotCellCodec) - -func toSlotCid*(hash: Poseidon2Hash): ?!Cid = - toCid(hash, multiCodec("identity"), SlotRootCodec) - -func toSlotCids*(slotRoots: openArray[Poseidon2Hash]): ?!seq[Cid] = - success slotRoots.mapIt(?it.toSlotCid) - -func fromSlotCid*(cid: Cid): ?!Poseidon2Hash = - toPoseidon2Hash(cid, multiCodec("identity"), SlotRootCodec) - -func toVerifyCid*(hash: Poseidon2Hash): ?!Cid = - toCid(hash, multiCodec("identity"), SlotProvingRootCodec) - -func fromVerifyCid*(cid: Cid): ?!Poseidon2Hash = - toPoseidon2Hash(cid, multiCodec("identity"), SlotProvingRootCodec) - -func toEncodableProof*(proof: Poseidon2Proof): ?!CodexProof = - let encodableProof = CodexProof( - mcodec: multiCodec("identity"), - index: proof.index, - nleaves: proof.nleaves, - path: proof.path.mapIt(@(it.toBytes)), - ) - - success encodableProof - -func toVerifiableProof*(proof: CodexProof): ?!Poseidon2Proof = - let nodes = proof.path.mapIt(?Poseidon2Hash.fromBytes(it.toArray32).toFailure) - - Poseidon2Proof.init(index = proof.index, nleaves = proof.nleaves, nodes = nodes) diff --git a/codex/slots/proofs.nim b/codex/slots/proofs.nim deleted file mode 100644 index 4f7f01b5..00000000 --- a/codex/slots/proofs.nim +++ /dev/null @@ -1,5 +0,0 @@ -import ./proofs/backends -import ./proofs/prover -import ./proofs/backendfactory - -export circomcompat, prover, backendfactory diff --git a/codex/slots/proofs/backendfactory.nim b/codex/slots/proofs/backendfactory.nim deleted file mode 100644 index 7aba27d8..00000000 --- a/codex/slots/proofs/backendfactory.nim +++ /dev/null @@ -1,82 +0,0 @@ -import os -import strutils -import pkg/chronos -import pkg/chronicles -import pkg/questionable -import pkg/confutils/defs -import pkg/stew/io2 -import pkg/ethers - -import ../../conf -import ./backends -import ./backendutils - -proc initializeFromConfig(config: CodexConf, utils: BackendUtils): ?!AnyBackend = - if not fileAccessible($config.circomR1cs, {AccessFlags.Read}) or - not endsWith($config.circomR1cs, ".r1cs"): - return failure("Circom R1CS file not accessible") - - if not fileAccessible($config.circomWasm, {AccessFlags.Read}) or - not endsWith($config.circomWasm, ".wasm"): - return failure("Circom wasm file not accessible") - - if not fileAccessible($config.circomZkey, {AccessFlags.Read}) or - not endsWith($config.circomZkey, ".zkey"): - return failure("Circom zkey file not accessible") - - trace "Initialized prover backend from cli config" - success( - utils.initializeCircomBackend( - $config.circomR1cs, $config.circomWasm, $config.circomZkey - ) - ) - -proc r1csFilePath(config: CodexConf): string = - config.circuitDir / "proof_main.r1cs" - -proc wasmFilePath(config: CodexConf): string = - config.circuitDir / "proof_main.wasm" - -proc zkeyFilePath(config: CodexConf): string = - config.circuitDir / "proof_main.zkey" - -proc initializeFromCircuitDirFiles( - config: CodexConf, utils: BackendUtils -): ?!AnyBackend {.gcsafe.} = - if fileExists(config.r1csFilePath) and fileExists(config.wasmFilePath) and - fileExists(config.zkeyFilePath): - trace "Initialized prover backend from local files" - return success( - utils.initializeCircomBackend( - config.r1csFilePath, config.wasmFilePath, config.zkeyFilePath - ) - ) - - failure("Circuit files not found") - -proc suggestDownloadTool(config: CodexConf) = - without address =? config.marketplaceAddress: - raise (ref Defect)( - msg: "Proving backend initializing while marketplace address not set." - ) - - let - tokens = ["cirdl", "\"" & $config.circuitDir & "\"", config.ethProvider, $address] - instructions = "'./" & tokens.join(" ") & "'" - - warn "Proving circuit files are not found. Please run the following to download them:", - instructions - -proc initializeBackend*( - config: CodexConf, utils: BackendUtils = BackendUtils() -): ?!AnyBackend = - without backend =? initializeFromConfig(config, utils), cliErr: - info "Could not initialize prover backend from CLI options...", msg = cliErr.msg - without backend =? initializeFromCircuitDirFiles(config, utils), localErr: - info "Could not initialize prover backend from circuit dir files...", - msg = localErr.msg - suggestDownloadTool(config) - return failure("CircuitFilesNotFound") - # Unexpected: value of backend does not survive leaving each scope. (definition does though...) - return success(backend) - return success(backend) diff --git a/codex/slots/proofs/backends.nim b/codex/slots/proofs/backends.nim deleted file mode 100644 index 3bd2edb6..00000000 --- a/codex/slots/proofs/backends.nim +++ /dev/null @@ -1,5 +0,0 @@ -import ./backends/circomcompat - -export circomcompat - -type AnyBackend* = CircomCompat diff --git a/codex/slots/proofs/backends/circomcompat.nim b/codex/slots/proofs/backends/circomcompat.nim deleted file mode 100644 index 479866f4..00000000 --- a/codex/slots/proofs/backends/circomcompat.nim +++ /dev/null @@ -1,240 +0,0 @@ -## Logos Storage -## Copyright (c) 2024 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [].} - -import std/sugar - -import pkg/chronos -import pkg/questionable/results -import pkg/circomcompat - -import ../../types -import ../../../stores -import ../../../contracts - -import ./converters - -export circomcompat, converters - -type - CircomCompat* = object - slotDepth: int # max depth of the slot tree - datasetDepth: int # max depth of dataset tree - blkDepth: int # depth of the block merkle tree (pow2 for now) - cellElms: int # number of field elements per cell - numSamples: int # number of samples per slot - r1csPath: string # path to the r1cs file - wasmPath: string # path to the wasm file - zkeyPath: string # path to the zkey file - backendCfg: ptr CircomBn254Cfg - vkp*: ptr CircomKey - - NormalizedProofInputs*[H] {.borrow: `.`.} = distinct ProofInputs[H] - -func normalizeInput*[H]( - self: CircomCompat, input: ProofInputs[H] -): NormalizedProofInputs[H] = - ## Parameters in CIRCOM circuits are statically sized and must be properly - ## padded before they can be passed onto the circuit. This function takes - ## variable length parameters and performs that padding. - ## - ## The output from this function can be JSON-serialized and used as direct - ## inputs to the CIRCOM circuit for testing and debugging when one wishes - ## to bypass the Rust FFI. - - let normSamples = collect: - for sample in input.samples: - var merklePaths = sample.merklePaths - merklePaths.setLen(self.slotDepth) - Sample[H](cellData: sample.cellData, merklePaths: merklePaths) - - var normSlotProof = input.slotProof - normSlotProof.setLen(self.datasetDepth) - - NormalizedProofInputs[H] ProofInputs[H]( - entropy: input.entropy, - datasetRoot: input.datasetRoot, - slotIndex: input.slotIndex, - slotRoot: input.slotRoot, - nCellsPerSlot: input.nCellsPerSlot, - nSlotsPerDataSet: input.nSlotsPerDataSet, - slotProof: normSlotProof, - samples: normSamples, - ) - -proc release*(self: CircomCompat) = - ## Release the ctx - ## - - if not isNil(self.backendCfg): - self.backendCfg.unsafeAddr.release_cfg() - - if not isNil(self.vkp): - self.vkp.unsafeAddr.release_key() - -proc prove[H](self: CircomCompat, input: NormalizedProofInputs[H]): ?!CircomProof = - doAssert input.samples.len == self.numSamples, "Number of samples does not match" - - doAssert input.slotProof.len <= self.datasetDepth, - "Slot proof is too deep - dataset has more slots than what we can handle?" - - doAssert input.samples.allIt( - block: - ( - it.merklePaths.len <= self.slotDepth + self.blkDepth and - it.cellData.len == self.cellElms - ) - ), "Merkle paths too deep or cells too big for circuit" - - # TODO: All parameters should match circom's static parametter - var ctx: ptr CircomCompatCtx - - defer: - if ctx != nil: - ctx.addr.release_circom_compat() - - if init_circom_compat(self.backendCfg, addr ctx) != ERR_OK or ctx == nil: - raiseAssert("failed to initialize CircomCompat ctx") - - var - entropy = input.entropy.toBytes - dataSetRoot = input.datasetRoot.toBytes - slotRoot = input.slotRoot.toBytes - - if ctx.push_input_u256_array("entropy".cstring, entropy[0].addr, entropy.len.uint32) != - ERR_OK: - return failure("Failed to push entropy") - - if ctx.push_input_u256_array( - "dataSetRoot".cstring, dataSetRoot[0].addr, dataSetRoot.len.uint32 - ) != ERR_OK: - return failure("Failed to push data set root") - - if ctx.push_input_u256_array( - "slotRoot".cstring, slotRoot[0].addr, slotRoot.len.uint32 - ) != ERR_OK: - return failure("Failed to push data set root") - - if ctx.push_input_u32("nCellsPerSlot".cstring, input.nCellsPerSlot.uint32) != ERR_OK: - return failure("Failed to push nCellsPerSlot") - - if ctx.push_input_u32("nSlotsPerDataSet".cstring, input.nSlotsPerDataSet.uint32) != - ERR_OK: - return failure("Failed to push nSlotsPerDataSet") - - if ctx.push_input_u32("slotIndex".cstring, input.slotIndex.uint32) != ERR_OK: - return failure("Failed to push slotIndex") - - var slotProof = input.slotProof.mapIt(it.toBytes).concat - - doAssert(slotProof.len == self.datasetDepth) - # arrays are always flattened - if ctx.push_input_u256_array( - "slotProof".cstring, slotProof[0].addr, uint (slotProof[0].len * slotProof.len) - ) != ERR_OK: - return failure("Failed to push slot proof") - - for s in input.samples: - var - merklePaths = s.merklePaths.mapIt(it.toBytes) - data = s.cellData.mapIt(@(it.toBytes)).concat - - if ctx.push_input_u256_array( - "merklePaths".cstring, - merklePaths[0].addr, - uint (merklePaths[0].len * merklePaths.len), - ) != ERR_OK: - return failure("Failed to push merkle paths") - - if ctx.push_input_u256_array("cellData".cstring, data[0].addr, data.len.uint) != - ERR_OK: - return failure("Failed to push cell data") - - var proofPtr: ptr Proof = nil - - let proof = - try: - if (let res = self.backendCfg.prove_circuit(ctx, proofPtr.addr); res != ERR_OK) or - proofPtr == nil: - return failure("Failed to prove - err code: " & $res) - - proofPtr[] - finally: - if proofPtr != nil: - proofPtr.addr.release_proof() - - success proof - -proc prove*[H](self: CircomCompat, input: ProofInputs[H]): ?!CircomProof = - self.prove(self.normalizeInput(input)) - -proc verify*[H]( - self: CircomCompat, proof: CircomProof, inputs: ProofInputs[H] -): ?!bool = - ## Verify a proof using a ctx - ## - - var - proofPtr = unsafeAddr proof - inputs = inputs.toCircomInputs() - - try: - let res = verify_circuit(proofPtr, inputs.addr, self.vkp) - if res == ERR_OK: - success true - elif res == ERR_FAILED_TO_VERIFY_PROOF: - success false - else: - failure("Failed to verify proof - err code: " & $res) - finally: - inputs.releaseCircomInputs() - -proc init*( - _: type CircomCompat, - r1csPath: string, - wasmPath: string, - zkeyPath: string = "", - slotDepth = DefaultMaxSlotDepth, - datasetDepth = DefaultMaxDatasetDepth, - blkDepth = DefaultBlockDepth, - cellElms = DefaultCellElms, - numSamples = DefaultSamplesNum, -): CircomCompat = - ## Create a new ctx - ## - - var cfg: ptr CircomBn254Cfg - var zkey = if zkeyPath.len > 0: zkeyPath.cstring else: nil - - if init_circom_config(r1csPath.cstring, wasmPath.cstring, zkey, cfg.addr) != ERR_OK or - cfg == nil: - if cfg != nil: - cfg.addr.release_cfg() - raiseAssert("failed to initialize circom compat config") - - var vkpPtr: ptr VerifyingKey = nil - - if cfg.get_verifying_key(vkpPtr.addr) != ERR_OK or vkpPtr == nil: - if vkpPtr != nil: - vkpPtr.addr.release_key() - raiseAssert("Failed to get verifying key") - - CircomCompat( - r1csPath: r1csPath, - wasmPath: wasmPath, - zkeyPath: zkeyPath, - slotDepth: slotDepth, - datasetDepth: datasetDepth, - blkDepth: blkDepth, - cellElms: cellElms, - numSamples: numSamples, - backendCfg: cfg, - vkp: vkpPtr, - ) diff --git a/codex/slots/proofs/backends/converters.nim b/codex/slots/proofs/backends/converters.nim deleted file mode 100644 index 19ea3b35..00000000 --- a/codex/slots/proofs/backends/converters.nim +++ /dev/null @@ -1,54 +0,0 @@ -## Logos Storage -## Copyright (c) 2024 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -{.push raises: [].} - -import pkg/circomcompat - -import ../../../contracts -import ../../types -import ../../../merkletree - -type - CircomG1* = G1 - CircomG2* = G2 - - CircomProof* = Proof - CircomKey* = VerifyingKey - CircomInputs* = Inputs - -proc toCircomInputs*(inputs: ProofInputs[Poseidon2Hash]): CircomInputs = - var - slotIndex = inputs.slotIndex.toF.toBytes.toArray32 - datasetRoot = inputs.datasetRoot.toBytes.toArray32 - entropy = inputs.entropy.toBytes.toArray32 - - elms = [entropy, datasetRoot, slotIndex] - - let inputsPtr = allocShared0(32 * elms.len) - copyMem(inputsPtr, addr elms[0], elms.len * 32) - - CircomInputs(elms: cast[ptr array[32, byte]](inputsPtr), len: elms.len.uint) - -proc releaseCircomInputs*(inputs: var CircomInputs) = - if not inputs.elms.isNil: - deallocShared(inputs.elms) - inputs.elms = nil - -func toG1*(g: CircomG1): G1Point = - G1Point(x: UInt256.fromBytesLE(g.x), y: UInt256.fromBytesLE(g.y)) - -func toG2*(g: CircomG2): G2Point = - G2Point( - x: Fp2Element(real: UInt256.fromBytesLE(g.x[0]), imag: UInt256.fromBytesLE(g.x[1])), - y: Fp2Element(real: UInt256.fromBytesLE(g.y[0]), imag: UInt256.fromBytesLE(g.y[1])), - ) - -func toGroth16Proof*(proof: CircomProof): Groth16Proof = - Groth16Proof(a: proof.a.toG1, b: proof.b.toG2, c: proof.c.toG1) diff --git a/codex/slots/proofs/backendutils.nim b/codex/slots/proofs/backendutils.nim deleted file mode 100644 index 0e334ace..00000000 --- a/codex/slots/proofs/backendutils.nim +++ /dev/null @@ -1,8 +0,0 @@ -import ./backends - -type BackendUtils* = ref object of RootObj - -method initializeCircomBackend*( - self: BackendUtils, r1csFile: string, wasmFile: string, zKeyFile: string -): AnyBackend {.base, gcsafe.} = - CircomCompat.init(r1csFile, wasmFile, zKeyFile) diff --git a/codex/slots/proofs/prover.nim b/codex/slots/proofs/prover.nim deleted file mode 100644 index b3e83d7c..00000000 --- a/codex/slots/proofs/prover.nim +++ /dev/null @@ -1,93 +0,0 @@ -## Logos Storage -## Copyright (c) 2024 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. -## - -import pkg/chronos -import pkg/chronicles -import pkg/circomcompat -import pkg/poseidon2 -import pkg/questionable/results - -import pkg/libp2p/cid - -import ../../manifest -import ../../merkletree -import ../../stores -import ../../market -import ../../utils/poseidon2digest -import ../../conf - -import ../builder -import ../sampler - -import ./backends -import ../types - -export backends - -logScope: - topics = "codex prover" - -type - AnyProof* = CircomProof - - AnySampler* = Poseidon2Sampler - # add any other generic type here, eg. Poseidon2Sampler | ReinforceConcreteSampler - AnyBuilder* = Poseidon2Builder - # add any other generic type here, eg. Poseidon2Builder | ReinforceConcreteBuilder - - AnyProofInputs* = ProofInputs[Poseidon2Hash] - Prover* = ref object of RootObj - backend: AnyBackend - store: BlockStore - nSamples: int - -proc prove*( - self: Prover, slotIdx: int, manifest: Manifest, challenge: ProofChallenge -): Future[?!(AnyProofInputs, AnyProof)] {.async: (raises: [CancelledError]).} = - ## Prove a statement using backend. - ## Returns a future that resolves to a proof. - - logScope: - cid = manifest.treeCid - slot = slotIdx - challenge = challenge - - trace "Received proof challenge" - - without builder =? AnyBuilder.new(self.store, manifest), err: - error "Unable to create slots builder", err = err.msg - return failure(err) - - without sampler =? AnySampler.new(slotIdx, self.store, builder), err: - error "Unable to create data sampler", err = err.msg - return failure(err) - - without proofInput =? await sampler.getProofInput(challenge, self.nSamples), err: - error "Unable to get proof input for slot", err = err.msg - return failure(err) - - # prove slot - without proof =? self.backend.prove(proofInput), err: - error "Unable to prove slot", err = err.msg - return failure(err) - - success (proofInput, proof) - -proc verify*( - self: Prover, proof: AnyProof, inputs: AnyProofInputs -): Future[?!bool] {.async: (raises: [CancelledError]).} = - ## Prove a statement using backend. - ## Returns a future that resolves to a proof. - self.backend.verify(proof, inputs) - -proc new*( - _: type Prover, store: BlockStore, backend: AnyBackend, nSamples: int -): Prover = - Prover(store: store, backend: backend, nSamples: nSamples) diff --git a/codex/slots/sampler.nim b/codex/slots/sampler.nim deleted file mode 100644 index 23cfb73f..00000000 --- a/codex/slots/sampler.nim +++ /dev/null @@ -1,8 +0,0 @@ -import ./sampler/sampler -import ./sampler/utils - -import ../merkletree - -export sampler, utils - -type Poseidon2Sampler* = DataSampler[Poseidon2Tree, Poseidon2Hash] diff --git a/codex/slots/sampler/sampler.nim b/codex/slots/sampler/sampler.nim deleted file mode 100644 index 1afae3fb..00000000 --- a/codex/slots/sampler/sampler.nim +++ /dev/null @@ -1,138 +0,0 @@ -## Logos Storage -## Copyright (c) 2023 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/sugar - -import pkg/chronos -import pkg/questionable -import pkg/questionable/results -import pkg/stew/arrayops - -import ../../logutils -import ../../market -import ../../blocktype as bt -import ../../merkletree -import ../../manifest -import ../../stores - -import ../converters -import ../builder -import ../types -import ./utils - -logScope: - topics = "codex datasampler" - -type DataSampler*[T, H] = ref object of RootObj - index: Natural - blockStore: BlockStore - builder: SlotsBuilder[T, H] - -func getCell*[T, H]( - self: DataSampler[T, H], blkBytes: seq[byte], blkCellIdx: Natural -): seq[H] = - let - cellSize = self.builder.cellSize.uint64 - dataStart = cellSize * blkCellIdx.uint64 - dataEnd = dataStart + cellSize - - doAssert (dataEnd - dataStart) == cellSize, "Invalid cell size" - - blkBytes[dataStart ..< dataEnd].elements(H).toSeq() - -proc getSample*[T, H]( - self: DataSampler[T, H], cellIdx: int, slotTreeCid: Cid, slotRoot: H -): Future[?!Sample[H]] {.async: (raises: [CancelledError]).} = - let - cellsPerBlock = self.builder.numBlockCells - blkCellIdx = cellIdx.toCellInBlk(cellsPerBlock) # block cell index - blkSlotIdx = cellIdx.toBlkInSlot(cellsPerBlock) # slot tree index - origBlockIdx = self.builder.slotIndices(self.index)[blkSlotIdx] - # convert to original dataset block index - - logScope: - cellIdx = cellIdx - blkSlotIdx = blkSlotIdx - blkCellIdx = blkCellIdx - origBlockIdx = origBlockIdx - - trace "Retrieving sample from block tree" - let - (_, proof) = (await self.blockStore.getCidAndProof(slotTreeCid, blkSlotIdx.Natural)).valueOr: - return failure("Failed to get slot tree CID and proof") - - slotProof = proof.toVerifiableProof().valueOr: - return failure("Failed to get verifiable proof") - - (bytes, blkTree) = (await self.builder.buildBlockTree(origBlockIdx, blkSlotIdx)).valueOr: - return failure("Failed to build block tree") - - cellData = self.getCell(bytes, blkCellIdx) - cellProof = blkTree.getProof(blkCellIdx).valueOr: - return failure("Failed to get proof from block tree") - - success Sample[H](cellData: cellData, merklePaths: (cellProof.path & slotProof.path)) - -proc getProofInput*[T, H]( - self: DataSampler[T, H], entropy: ProofChallenge, nSamples: Natural -): Future[?!ProofInputs[H]] {.async: (raises: [CancelledError]).} = - ## Generate proofs as input to the proving circuit. - ## - - let - entropy = H.fromBytes(array[31, byte].initCopyFrom(entropy[0 .. 30])) - # truncate to 31 bytes, otherwise it _might_ be greater than mod - - verifyTree = self.builder.verifyTree.toFailure.valueOr: - return failure("Failed to get verify tree") - - slotProof = verifyTree.getProof(self.index).valueOr: - return failure("Failed to get slot proof") - - datasetRoot = verifyTree.root().valueOr: - return failure("Failed to get dataset root") - - slotTreeCid = self.builder.manifest.slotRoots[self.index] - slotRoot = self.builder.slotRoots[self.index] - cellIdxs = entropy.cellIndices(slotRoot, self.builder.numSlotCells, nSamples) - - logScope: - cells = cellIdxs - - trace "Collecting input for proof" - let samples = collect(newSeq): - for cellIdx in cellIdxs: - (await self.getSample(cellIdx, slotTreeCid, slotRoot)).valueOr: - return failure("Failed to get sample") - - success ProofInputs[H]( - entropy: entropy, - datasetRoot: datasetRoot, - slotProof: slotProof.path, - nSlotsPerDataSet: self.builder.numSlots, - nCellsPerSlot: self.builder.numSlotCells, - slotRoot: slotRoot, - slotIndex: self.index, - samples: samples, - ) - -proc new*[T, H]( - _: type DataSampler[T, H], - index: Natural, - blockStore: BlockStore, - builder: SlotsBuilder[T, H], -): ?!DataSampler[T, H] = - if index > builder.slotRoots.high: - error "Slot index is out of range" - return failure("Slot index is out of range") - - if not builder.verifiable: - return failure("Cannot instantiate DataSampler for non-verifiable builder") - - success DataSampler[T, H](index: index, blockStore: blockStore, builder: builder) diff --git a/codex/slots/sampler/utils.nim b/codex/slots/sampler/utils.nim deleted file mode 100644 index 0e7c5b93..00000000 --- a/codex/slots/sampler/utils.nim +++ /dev/null @@ -1,75 +0,0 @@ -## Logos Storage -## Copyright (c) 2024 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/bitops - -import pkg/questionable/results -import pkg/constantine/math/arithmetic - -import ../../merkletree - -func extractLowBits*[n: static int](elm: BigInt[n], k: int): uint64 = - doAssert(k > 0 and k <= 64) - var r = 0'u64 - for i in 0 ..< k: - let b = bit[n](elm, i) - let y = uint64(b) - if (y != 0): - r = bitor(r, 1'u64 shl i) - r - -func extractLowBits(fld: Poseidon2Hash, k: int): uint64 = - let elm: BigInt[254] = fld.toBig() - return extractLowBits(elm, k) - -func floorLog2*(x: int): int = - doAssert (x > 0) - var k = -1 - var y = x - while (y > 0): - k += 1 - y = y shr 1 - return k - -func ceilingLog2*(x: int): int = - doAssert (x > 0) - return (floorLog2(x - 1) + 1) - -func toBlkInSlot*(cell: Natural, numCells: Natural): Natural = - let log2 = ceilingLog2(numCells) - doAssert(1 shl log2 == numCells, "`numCells` is assumed to be a power of two") - - return cell div numCells - -func toCellInBlk*(cell: Natural, numCells: Natural): Natural = - let log2 = ceilingLog2(numCells) - doAssert(1 shl log2 == numCells, "`numCells` is assumed to be a power of two") - - return cell mod numCells - -func cellIndex*( - entropy: Poseidon2Hash, slotRoot: Poseidon2Hash, numCells: Natural, counter: Natural -): Natural = - let log2 = ceilingLog2(numCells) - doAssert(1 shl log2 == numCells, "`numCells` is assumed to be a power of two") - - let hash = Sponge.digest(@[entropy, slotRoot, counter.toF], rate = 2) - return int(extractLowBits(hash, log2)) - -func cellIndices*( - entropy: Poseidon2Hash, - slotRoot: Poseidon2Hash, - numCells: Natural, - nSamples: Natural, -): seq[Natural] = - var indices: seq[Natural] - for i in 1 .. nSamples: - indices.add(cellIndex(entropy, slotRoot, numCells, i)) - - indices diff --git a/codex/slots/types.nim b/codex/slots/types.nim deleted file mode 100644 index 5976c9b0..00000000 --- a/codex/slots/types.nim +++ /dev/null @@ -1,30 +0,0 @@ -## Logos Storage -## Copyright (c) 2024 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -type - Sample*[H] = object - cellData*: seq[H] - merklePaths*: seq[H] - - PublicInputs*[H] = object - slotIndex*: int - datasetRoot*: H - entropy*: H - - ProofInputs*[H] = object - entropy*: H - datasetRoot*: H - slotIndex*: Natural - slotRoot*: H - nCellsPerSlot*: Natural - nSlotsPerDataSet*: Natural - slotProof*: seq[H] - # inclusion proof that shows that the slot root (leaf) is part of the dataset (root) - samples*: seq[Sample[H]] - # inclusion proofs which show that the selected cells (leafs) are part of the slot (roots) diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index af747dd6..d2fc51e9 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -54,8 +54,7 @@ proc new*( method `size`*(self: StoreStream): int = ## The size of a StoreStream is the size of the original dataset, without ## padding or parity blocks. - let m = self.manifest - (if m.protected: m.originalDatasetSize else: m.datasetSize).int + self.manifest.datasetSize.int proc `size=`*(self: StoreStream, size: int) {.error: "Setting the size is forbidden".} = discard diff --git a/codex/validation.nim b/codex/validation.nim deleted file mode 100644 index d9f8fb5e..00000000 --- a/codex/validation.nim +++ /dev/null @@ -1,151 +0,0 @@ -import std/sets -import std/sequtils -import pkg/chronos -import pkg/questionable/results -import pkg/stew/endians2 - -import ./validationconfig -import ./market -import ./clock -import ./logutils - -export market -export sets -export validationconfig - -type Validation* = ref object - slots: HashSet[SlotId] - clock: Clock - market: Market - subscriptions: seq[Subscription] - running: Future[void] - periodicity: Periodicity - proofTimeout: uint64 - config: ValidationConfig - -logScope: - topics = "codex validator" - -proc new*( - _: type Validation, clock: Clock, market: Market, config: ValidationConfig -): Validation = - Validation(clock: clock, market: market, config: config) - -proc slots*(validation: Validation): seq[SlotId] = - validation.slots.toSeq - -proc getCurrentPeriod(validation: Validation): Period = - return validation.periodicity.periodOf(validation.clock.now().Timestamp) - -proc waitUntilNextPeriod(validation: Validation) {.async.} = - let period = validation.getCurrentPeriod() - let periodEnd = validation.periodicity.periodEnd(period) - trace "Waiting until next period", currentPeriod = period - await validation.clock.waitUntil((periodEnd + 1).toSecondsSince1970) - -func groupIndexForSlotId*(slotId: SlotId, validationGroups: ValidationGroups): uint16 = - let a = slotId.toArray - let slotIdInt64 = uint64.fromBytesBE(a) - (slotIdInt64 mod uint64(validationGroups)).uint16 - -func maxSlotsConstraintRespected(validation: Validation): bool = - validation.config.maxSlots == 0 or validation.slots.len < validation.config.maxSlots - -func shouldValidateSlot(validation: Validation, slotId: SlotId): bool = - without validationGroups =? validation.config.groups: - return true - groupIndexForSlotId(slotId, validationGroups) == validation.config.groupIndex - -proc subscribeSlotFilled(validation: Validation) {.async.} = - proc onSlotFilled(requestId: RequestId, slotIndex: uint64) = - if not validation.maxSlotsConstraintRespected: - return - let slotId = slotId(requestId, slotIndex) - if validation.shouldValidateSlot(slotId): - trace "Adding slot", slotId - validation.slots.incl(slotId) - - let subscription = await validation.market.subscribeSlotFilled(onSlotFilled) - validation.subscriptions.add(subscription) - -proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = - var ended: HashSet[SlotId] - let slots = validation.slots - for slotId in slots: - let state = await validation.market.slotState(slotId) - if state != SlotState.Filled: - trace "Removing slot", slotId, slotState = state - ended.incl(slotId) - validation.slots.excl(ended) - -proc markProofAsMissing( - validation: Validation, slotId: SlotId, period: Period -) {.async: (raises: [CancelledError]).} = - logScope: - currentPeriod = validation.getCurrentPeriod() - - try: - if await validation.market.canMarkProofAsMissing(slotId, period): - trace "Marking proof as missing", slotId, periodProofMissed = period - await validation.market.markProofAsMissing(slotId, period) - else: - let inDowntime {.used.} = await validation.market.inDowntime(slotId) - trace "Proof not missing", checkedPeriod = period, inDowntime - except CancelledError as e: - raise e - except CatchableError as e: - error "Marking proof as missing failed", msg = e.msg - -proc markProofsAsMissing(validation: Validation) {.async: (raises: [CancelledError]).} = - let slots = validation.slots - for slotId in slots: - let previousPeriod = validation.getCurrentPeriod() - 1 - await validation.markProofAsMissing(slotId, previousPeriod) - -proc run(validation: Validation) {.async: (raises: [CancelledError]).} = - trace "Validation started" - try: - while true: - await validation.waitUntilNextPeriod() - await validation.removeSlotsThatHaveEnded() - await validation.markProofsAsMissing() - except CancelledError: - trace "Validation stopped" - discard # do not propagate as run is asyncSpawned - except CatchableError as e: - error "Validation failed", msg = e.msg - -proc findEpoch(validation: Validation, secondsAgo: uint64): SecondsSince1970 = - return validation.clock.now - secondsAgo.int64 - -proc restoreHistoricalState(validation: Validation) {.async.} = - trace "Restoring historical state..." - let requestDurationLimit = await validation.market.requestDurationLimit - let startTimeEpoch = validation.findEpoch(secondsAgo = requestDurationLimit) - let slotFilledEvents = - await validation.market.queryPastSlotFilledEvents(fromTime = startTimeEpoch) - for event in slotFilledEvents: - if not validation.maxSlotsConstraintRespected: - break - let slotId = slotId(event.requestId, event.slotIndex) - let slotState = await validation.market.slotState(slotId) - if slotState == SlotState.Filled and validation.shouldValidateSlot(slotId): - trace "Adding slot [historical]", slotId - validation.slots.incl(slotId) - trace "Historical state restored", numberOfSlots = validation.slots.len - -proc start*(validation: Validation) {.async.} = - trace "Starting validator", - groups = validation.config.groups, groupIndex = validation.config.groupIndex - validation.periodicity = await validation.market.periodicity() - validation.proofTimeout = await validation.market.proofTimeout() - await validation.subscribeSlotFilled() - await validation.restoreHistoricalState() - validation.running = validation.run() - -proc stop*(validation: Validation) {.async.} = - if not validation.running.isNil and not validation.running.finished: - await validation.running.cancelAndWait() - while validation.subscriptions.len > 0: - let subscription = validation.subscriptions.pop() - await subscription.unsubscribe() diff --git a/codex/validationconfig.nim b/codex/validationconfig.nim deleted file mode 100644 index 3e21c4fa..00000000 --- a/codex/validationconfig.nim +++ /dev/null @@ -1,35 +0,0 @@ -import std/strformat -import pkg/questionable -import pkg/questionable/results - -type - ValidationGroups* = range[2 .. 65535] - MaxSlots* = int - ValidationConfig* = object - maxSlots: MaxSlots - groups: ?ValidationGroups - groupIndex: uint16 - -func init*( - _: type ValidationConfig, - maxSlots: MaxSlots, - groups: ?ValidationGroups, - groupIndex: uint16 = 0, -): ?!ValidationConfig = - if maxSlots < 0: - return failure "The value of maxSlots must be greater than " & - fmt"or equal to 0! (got: {maxSlots})" - if validationGroups =? groups and groupIndex >= uint16(validationGroups): - return failure "The value of the group index must be less than " & - fmt"validation groups! (got: {groupIndex = }, " & fmt"groups = {validationGroups})" - - success ValidationConfig(maxSlots: maxSlots, groups: groups, groupIndex: groupIndex) - -func maxSlots*(config: ValidationConfig): MaxSlots = - config.maxSlots - -func groups*(config: ValidationConfig): ?ValidationGroups = - config.groups - -func groupIndex*(config: ValidationConfig): uint16 = - config.groupIndex diff --git a/library/storage_context.nim b/library/storage_context.nim index 260b6a98..718aaec5 100644 --- a/library/storage_context.nim +++ b/library/storage_context.nim @@ -59,7 +59,7 @@ template callEventCallback(ctx: ptr StorageContext, eventName: string, body: unt ## Template used to notify the client of global events ## Example: onConnectionChanged, onProofMissing, etc. if isNil(ctx[].eventCallback): - error eventName & " - eventCallback is nil" + error eventName&" - eventCallback is nil" return foreignThreadGc: diff --git a/library/storage_thread_requests/requests/node_lifecycle_request.nim b/library/storage_thread_requests/requests/node_lifecycle_request.nim index 2963f795..c04bd5bf 100644 --- a/library/storage_thread_requests/requests/node_lifecycle_request.nim +++ b/library/storage_thread_requests/requests/node_lifecycle_request.nim @@ -69,9 +69,6 @@ proc readValue*(r: var JsonReader, val: var Duration) = raise newException(SerializationError, "Cannot parse the duration: " & input) val = dur -proc readValue*(r: var JsonReader, val: var EthAddress) = - val = EthAddress.init(r.readValue(string)).get() - type NodeLifecycleRequest* = object operation: NodeLifecycleMsgType configJson: cstring diff --git a/openapi.yaml b/openapi.yaml index 5922f2c6..04d8960d 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -32,42 +32,11 @@ components: description: Content Identifier as specified at https://github.com/multiformats/cid example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N - SlotId: - type: string - description: Keccak hash of the abi encoded tuple (RequestId, slot index) - example: 268a781e0db3f7cf36b18e5f4fdb7f586ec9edd08e5500b17c0e518a769f114a - LogLevel: type: string description: "One of the log levels: TRACE, DEBUG, INFO, NOTICE, WARN, ERROR or FATAL" example: DEBUG - EthereumAddress: - type: string - description: Address of Ethereum address - - PricePerBytePerSecond: - type: string - description: The amount of tokens paid per byte per second per slot to hosts the client is willing to pay - - CollateralPerByte: - type: string - description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots - - Duration: - type: integer - format: int64 - description: The duration of the request in seconds - - ProofProbability: - type: string - description: How often storage proofs are required as decimal string - - Expiry: - type: integer - format: int64 - description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data. - SPR: type: string description: Signed Peer Record (libp2p) @@ -86,15 +55,6 @@ components: id: $ref: "#/components/schemas/PeerId" - Content: - type: object - required: - - cid - description: Parameters specifying the content - properties: - cid: - $ref: "#/components/schemas/Cid" - Node: type: object required: @@ -124,9 +84,6 @@ components: revision: type: string example: 0c647d8 - contracts: - type: string - example: 0b537c7 PeersTable: type: object @@ -172,251 +129,6 @@ components: storage: $ref: "#/components/schemas/StorageVersion" - SalesAvailability: - type: object - required: - - totalSize - - duration - - minPricePerBytePerSecond - - totalCollateral - properties: - totalSize: - type: integer - format: int64 - description: Total size of availability's storage in bytes - duration: - $ref: "#/components/schemas/Duration" - minPricePerBytePerSecond: - type: string - description: Minimal price per byte per second paid (in amount of tokens) for the hosted request's slot for the request's duration as decimal string - totalCollateral: - type: string - description: Total collateral (in amount of tokens) that can be used for matching requests - enabled: - type: boolean - description: Enable the ability to receive sales on this availability. - default: true - until: - type: integer - description: Specifies the latest timestamp, after which the availability will no longer host any slots. If set to 0, there will be no restrictions. - default: 0 - - SalesAvailabilityREAD: - required: - - id - - totalRemainingCollateral - - freeSize - allOf: - - $ref: "#/components/schemas/SalesAvailability" - - type: object - properties: - id: - $ref: "#/components/schemas/Id" - readonly: true - freeSize: - type: integer - format: int64 - description: Unused size of availability's storage in bytes as decimal string - readOnly: true - totalRemainingCollateral: - type: string - description: Total collateral effective (in amount of tokens) that can be used for matching requests - readOnly: true - - Slot: - type: object - required: - - id - - request - - slotIndex - properties: - id: - $ref: "#/components/schemas/SlotId" - request: - $ref: "#/components/schemas/StorageRequest" - slotIndex: - type: integer - format: int64 - description: Slot Index number - - SlotAgent: - type: object - required: - - state - - requestId - - slotIndex - properties: - slotIndex: - type: integer - format: int64 - description: Slot Index number - requestId: - $ref: "#/components/schemas/Id" - request: - $ref: "#/components/schemas/StorageRequest" - reservation: - $ref: "#/components/schemas/Reservation" - state: - type: string - description: Description of the slot's - enum: - - SaleCancelled - - SaleDownloading - - SaleErrored - - SaleFailed - - SaleFilled - - SaleFilling - - SaleFinished - - SaleIgnored - - SaleInitialProving - - SalePayout - - SalePreparing - - SaleProving - - SaleUnknown - - Reservation: - type: object - required: - - id - - availabilityId - - size - - requestId - - slotIndex - - validUntil - properties: - id: - $ref: "#/components/schemas/Id" - availabilityId: - $ref: "#/components/schemas/Id" - size: - type: integer - format: int64 - description: Size of the slot in bytes - requestId: - $ref: "#/components/schemas/Id" - slotIndex: - type: integer - format: int64 - description: Slot Index number - validUntil: - type: integer - description: Timestamp after which the reservation will no longer be valid. - - StorageRequestCreation: - type: object - required: - - pricePerBytePerSecond - - duration - - proofProbability - - collateralPerByte - - expiry - properties: - duration: - $ref: "#/components/schemas/Duration" - pricePerBytePerSecond: - $ref: "#/components/schemas/PricePerBytePerSecond" - proofProbability: - $ref: "#/components/schemas/ProofProbability" - nodes: - description: Minimal number of nodes the content should be stored on - type: integer - default: 3 - minimum: 3 - tolerance: - description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost - type: integer - default: 1 - minimum: 1 - collateralPerByte: - $ref: "#/components/schemas/CollateralPerByte" - expiry: - type: integer - format: int64 - description: Number that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself. - StorageAsk: - type: object - required: - - slots - - slotSize - - duration - - proofProbability - - pricePerBytePerSecond - - collateralPerByte - - maxSlotLoss - properties: - slots: - description: Number of slots (eq. hosts) that the Request want to have the content spread over - type: integer - format: int64 - slotSize: - type: integer - format: int64 - description: Amount of storage per slot in bytes - duration: - $ref: "#/components/schemas/Duration" - proofProbability: - $ref: "#/components/schemas/ProofProbability" - pricePerBytePerSecond: - $ref: "#/components/schemas/PricePerBytePerSecond" - collateralPerByte: - $ref: "#/components/schemas/CollateralPerByte" - maxSlotLoss: - type: integer - format: int64 - description: Max slots that can be lost without data considered to be lost - - StorageRequest: - type: object - required: - - id - - client - - ask - - content - - expiry - - nonce - properties: - id: - type: string - description: Request ID - client: - $ref: "#/components/schemas/EthereumAddress" - ask: - $ref: "#/components/schemas/StorageAsk" - content: - $ref: "#/components/schemas/Content" - expiry: - $ref: "#/components/schemas/Expiry" - nonce: - type: string - description: Random data - - Purchase: - type: object - required: - - state - - requestId - properties: - state: - type: string - description: Description of the Request's state - enum: - - cancelled - - errored - - failed - - finished - - pending - - started - - submitted - - unknown - error: - type: string - nullable: true - description: If Request failed, then here is presented the error message - request: - $ref: "#/components/schemas/StorageRequest" - requestId: - $ref: "#/components/schemas/Id" - DataList: type: object required: @@ -444,7 +156,6 @@ components: - treeCid - datasetSize - blockSize - - protected properties: treeCid: $ref: "#/components/schemas/Cid" @@ -456,9 +167,6 @@ components: blockSize: type: integer description: "Size of blocks" - protected: - type: boolean - description: "Indicates if content is protected by erasure-coding" filename: type: string nullable: true @@ -499,8 +207,6 @@ servers: - url: "http://localhost:8080/api/storage/v1" tags: - - name: Marketplace - description: Marketplace information and operations - name: Data description: Data operations - name: Node @@ -771,237 +477,6 @@ paths: "500": description: "It's not working as planned" - "/sales/slots": - get: - summary: "Returns active slots" - tags: [Marketplace] - operationId: getActiveSlots - responses: - "200": - description: Retrieved active slots - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/Slot" - - "503": - description: Persistence is not enabled - - "/sales/slots/{slotId}": - get: - summary: "Returns active slot with id {slotId} for the host" - tags: [Marketplace] - operationId: getActiveSlotById - parameters: - - in: path - name: slotId - required: true - schema: - $ref: "#/components/schemas/Cid" - description: File to be downloaded. - responses: - "200": - description: Retrieved active slot - content: - application/json: - schema: - $ref: "#/components/schemas/SlotAgent" - - "400": - description: Invalid or missing SlotId - - "404": - description: Host is not in an active sale for the slot - - "503": - description: Persistence is not enabled - - "/sales/availability": - get: - summary: "Returns storage that is for sale" - tags: [Marketplace] - operationId: getAvailabilities - responses: - "200": - description: Retrieved storage availabilities of the node - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/SalesAvailabilityREAD" - "500": - description: Error getting unused availabilities - "503": - description: Persistence is not enabled - - post: - summary: "Offers storage for sale" - operationId: offerStorage - tags: [Marketplace] - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/SalesAvailability" - responses: - "201": - description: Created storage availability - content: - application/json: - schema: - $ref: "#/components/schemas/SalesAvailabilityREAD" - "400": - description: Invalid data input - "422": - description: Not enough node's storage quota available or the provided parameters did not pass validation - "500": - description: Error reserving availability - "503": - description: Persistence is not enabled - "/sales/availability/{id}": - patch: - summary: "Updates availability" - description: | - The new parameters will be only considered for new requests. - Existing Requests linked to this Availability will continue as is. - operationId: updateOfferedStorage - tags: [Marketplace] - parameters: - - in: path - name: id - required: true - schema: - type: string - description: ID of Availability - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/SalesAvailability" - responses: - "204": - description: Availability successfully updated - "400": - description: Invalid data input - "404": - description: Availability not found - "422": - description: The provided parameters did not pass validation - "500": - description: Error reserving availability - "503": - description: Persistence is not enabled - - "/sales/availability/{id}/reservations": - get: - summary: "Get availability's reservations" - description: Return's list of Reservations for ongoing Storage Requests that the node hosts. - operationId: getReservations - tags: [Marketplace] - parameters: - - in: path - name: id - required: true - schema: - type: string - description: ID of Availability - responses: - "200": - description: Retrieved storage availabilities of the node - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/Reservation" - "400": - description: Invalid Availability ID - "404": - description: Availability not found - "500": - description: Error getting reservations - "503": - description: Persistence is not enabled - - "/storage/request/{cid}": - post: - summary: "Creates a new Request for storage" - tags: [Marketplace] - operationId: createStorageRequest - parameters: - - in: path - name: cid - required: true - schema: - $ref: "#/components/schemas/Cid" - description: CID of the uploaded data that should be stored - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/StorageRequestCreation" - responses: - "200": - description: Returns the Request ID as decimal string - content: - text/plain: - schema: - type: string - "400": - description: Invalid or missing Request ID - "422": - description: The storage request parameters are not valid - "404": - description: Request ID not found - "503": - description: Persistence is not enabled - - "/storage/purchases": - get: - summary: "Returns list of purchase IDs" - tags: [Marketplace] - operationId: getPurchases - responses: - "200": - description: Gets all purchase IDs stored in node - content: - application/json: - schema: - type: array - items: - type: string - "503": - description: Persistence is not enabled - - "/storage/purchases/{id}": - get: - summary: "Returns purchase details" - tags: [Marketplace] - operationId: getPurchase - parameters: - - in: path - name: id - required: true - schema: - type: string - description: Hexadecimal ID of a Purchase - responses: - "200": - description: Purchase details - content: - application/json: - schema: - $ref: "#/components/schemas/Purchase" - "400": - description: Invalid or missing Purchase ID - "404": - description: Purchase not found - "503": - description: Persistence is not enabled - "/spr": get: summary: "Get Node's SPR" diff --git a/tests/circuits/fixtures/input.json b/tests/circuits/fixtures/input.json deleted file mode 100644 index 0de9bcfc..00000000 --- a/tests/circuits/fixtures/input.json +++ /dev/null @@ -1,527 +0,0 @@ -{ - "dataSetRoot": "16074246370508166450132968585287196391860062495017081813239200574579640171677" -, "entropy": "1234567" -, "nCellsPerSlot": 512 -, "nSlotsPerDataSet": 11 -, "slotIndex": 3 -, "slotRoot": "20744935707483803411869804102043283881376973626291244537230284476834672019997" -, "slotProof": - [ "14279309641024220656349577745390262299143357053971618723978902485113885925133" - , "17350220251883387610715080716935498684002984280929482268590417788651882821293" - , "3614172556528990402229172918446087216573760062512459539027101853103043539066" - , "9593656216696187567506330076677122799107266567595923589824071605501987205034" - , "0" - , "0" - , "0" - , "0" - ] -, "cellData": - [ [ "211066599696340205996365563960462032209214145564176017965177408819390441927" - , "256399834032317991719099034134771774537377713676282398278615627599320306708" - , "40526956212941839024868120947422067322935297516255336725720469887577875470" - , "369406626072040375689238003388146123438765868500054546379159741776926336393" - , "333671948877941061129138970028865844558745266314244224980179694307884999701" - , "12844191661993811401197260475054004253686019503294245287625435635597431176" - , "103242505924551040184986153577926463300854479121445566984435820844932904541" - , "357267598134410031301285377503939045679462829143562392803919346036584141082" - , "162961530392479745020607594774288130869393650464001426863668385541077786641" - , "426633666684068667053061341108297711407154520752589357873877687002123230254" - , "131217200633679697678620573903653316618851166977399592012996945149000115543" - , "347806146382240882424437318028620747118202052207615339887983883245341422889" - , "373560143578047669373240014805175743607410583882636003337120578008437374619" - , "188643112726610812698950916978832639394489168469865816132268079928321994342" - , "261643693073361806247543578456646407192325702146752104760117340650316255422" - , "260425332276127964154119199351807753107358504026338378706506655351595199132" - , "374895089121103633563000003194929058314955925591115574099805066048344387554" - , "251251687633538360627384151887287228926166832908633974282160065378311171354" - , "72870348025150463132527129203816383011245664502016571773976961034605631401" - , "234969517550818492643515432666311755833657223377594670026839098818269671638" - , "250704662734070531273640113211555977086430125469327371276827055724111014200" - , "85287059658255939741261887818611116570376488685014337052369839580946343903" - , "148959658976765873884541474400081762855732313547557243964921157967555048302" - , "402116967301520959272239788104745348344918207829380669065019055837727389479" - , "440503687192139964066297025050080823601005280790824714962651368433530759519" - , "149064344353438643307231355233484617851197634669308957706338957575177745645" - , "249140840255377018814348914718469942883867200392561109698520706525194687651" - , "108796851176515124780842490199733462942992752881710253277179665118758071359" - , "168245155425564161902686596247453762364112240129335852645432169468767513906" - , "129930759770229612264501691941454447321585806001002088749773920103899362070" - , "26204732465033162738545933008873662562758704699080684615280202127289894343" - , "434343986187775289860542785529657444690073100436887472033336117760907652966" - , "361202432740487795596808128962740911600093857340619816047190021218849540225" - , "100616813001075101816809823591611435583084916492802624686700765550893945525" - , "262383566766515164611427346701355047932794351290691325516723194829671679460" - , "223966317895663049002893008659178463136086169222436544014405848127792334099" - , "416071800998333357259662686053338495720384342746822618737948080251761863079" - , "402582378631671531909245563300554883898015688826468151075059467077182712018" - , "271682867846395286938993638577506552857925968097084028550962231439839229096" - , "447239701195120035004067146414333903010840427278848992921567785105217019890" - , "275718367441702584521943480326858762208121719038684001399322597215141179102" - , "86424132534636958411139699704035287483951667275825975536719519441147485752" - , "149332313586183975744469174256094358432564607635406143904268565140988616920" - , "431284330776421588418608279008210842837281123158951642164956884286883748089" - , "328694410745471749523135644660455669483988686888634622076863114197617693825" - , "112671940998917362968156144648543607958275336559773039070338509488400224090" - , "40612851250697989627190554726382690498263128439797780029697069621854862060" - , "235047914228675997216342104196597257178021277585839376175077878186492271543" - , "169718735151210135199527910197065439221144015957220768545119706561079163228" - , "345850109040121443591415752965486014695272086901102608769402892906795715635" - , "107916794601837835951508003838161872232087225679609623116098837565956752373" - , "415195406060373162425374246348197423165008252112453298469572523506488563795" - , "18574536720926634955170276058049993354780447816096707123565164996905722992" - , "77316964375201096801231570737992491072607736322255206149311341101525354519" - , "198566211140075666401818378905444403588640345933666108724809349396921957675" - , "71926050707400318807625168942501384117254391471312636171205913503928815127" - , "303403754792341398295052586417779761162194818661412867094096550880325459639" - , "444796230931706624375881141460151785952798771079111017071124833045933389733" - , "430832727519144759265424205289522948157007336118070755365887670730658782114" - , "75431213985866235726407973044434444984663930761207296437571668004273515965" - , "9242635103653159191249730220870735855877366952081272723035956668095954838" - , "93770678769846326584848478152412123903909949977598807336203128684179492141" - , "438043261966019084676377174988310087831395864304423411701911688757793135582" - , "175357918416563657734972138036003712114814934655676571874083109097142591069" - , "301619681954194702458985259161884119574424456150215738560639417824784261940" - , "376627771252167062559065889174364784087884871999807562320457079200311413098" - , "77407" - ] - , [ "424838222746158988229624788694939151178615656210585621868910231014323837551" - , "113188486246955346418956949679485575685258850346101035778277727456423482970" - , "275449978206132565019222655023969430014622832597654643418394602485645803413" - , "407856757018138010439232009766252068440920591566039673660874626878413077905" - , "433369046493777496016384947949950877133856528218602671493669395706908819748" - , "258364166531180422149545015891786023981872586904946376136311798402581278793" - , "111997719221028147522803956659709775148434180015507797582340706052412284571" - , "370086597976426508280413876667101417393598181708677558733730556109327409076" - , "394139601979449259075910238117153992797849553309541269624848742084205563806" - , "224088276319080487199395482893988152025671468715318212801266537036640477323" - , "412710245119809501914481942088314642684754542082140451180970198371889738885" - , "353872602359946553306242341162348980635834907495492814598834657175405697176" - , "252575199506028475372678621140654219936768774012823764047177692104580452933" - , "259093153824033122452869556249315839899366493071746219770487886456301642099" - , "433829976798312333371154167497560676982294392632725612538640639617101218872" - , "69918581382122563555200898078544150952625715196904114153232367538572342772" - , "337937520623192257595352158476909569245988839238498098464935654555688460123" - , "264739973232292969253318643276671532055689422253184191167449284055983944338" - , "326116252818829775096345069850111970510714050346103409479803743342138260656" - , "236666502377413649728378889488706275212721356921124776708954261777813709815" - , "211625935799984260567718228446525455893664313064841539301444509150157287163" - , "60213206239417039999880027112341976360540689886703427811513517396638607512" - , "68310118105957780876770075529546844404225720757669797609686816545988561625" - , "423863085551351065136684030270731679105571943009795949621903966660399419936" - , "388914614294393005039878123500859325222684672184567792659076815268598434245" - , "449456790291560508709069826219925144971979653209755565240911568965768874382" - , "448810363770763694447869940916735951256986784286793489549428379909616059117" - , "93646909783664049092056237949587618925209622020026157405117796611689551192" - , "352210795298632954574896499649181574584074853828419384742874364724522457331" - , "37455517056393404525863484733101879886413925183061645520768912552476716150" - , "386617357584684336812125385078476270301738184058813703112840991226785114117" - , "309940292044597334261558429176136686101590729982259514472573656131579113438" - , "375815246167575100319857872432879650174355611853064771241069582477717074415" - , "332214507344122806007757734266883566559371568252728459951766124888176633706" - , "148990259460952914990881100852534318351247069504848477833147446514732789712" - , "328669527889838880414072022433859139004058211332184916573516704632073044118" - , "39278026039348543645873027549112998051601664395028652771103624299930924528" - , "147717660530589785119644237145092759103012624422229579698752386490700965238" - , "374018518701362017594752095877197725242352803195413267746619111489936980685" - , "19185486483883210730969367354195688373879769005461272861759636600984416877" - , "61866046506558157021682973167090213780467780519546382332208868591026703563" - , "186854966504766517012887726614015646154225796572138017810371160981778288347" - , "87813550250328892091332566928942844770632705056120813488729800874811845697" - , "207775163424060266085108794048371834145545842567796157378282772999548202308" - , "369987573847786237689249538753881486995686208870889713177975415012214427429" - , "240880979016395044518849230927615466120209140082149273390921042537474853143" - , "174902051454932913375934735427101804474275543418199101786687925733405159872" - , "342217255652950822372803598682842961053537267723988087801275319754065261308" - , "403207518067666945448161377960706451817747922771285796668778802535227939962" - , "407191459999036791052261270163259267557900498930952521056725210031161568230" - , "338216472523551728793268225845396218561132966393725938551091882807069657206" - , "118364222678576550870375112494142500603091119946985793934499723872824782886" - , "269721611028665323587192624288165848310917029418261851436925829954710472436" - , "227424498125745236125352117206136621428869662458452610254773560636280935711" - , "334380807433339401906359555583987757411855090694162252260781648609761248049" - , "42470806516174819075107446234449247453971524726021445768611797530804156161" - , "418994916402918322951830716592888390611524984156817012683478842068581638820" - , "363263142412048420546716019774090729399081311227606555141174736853886128407" - , "192292715923468025058557166341646729623133127372303791236447550026886802680" - , "450253878713722337767292128303865371116770532625906889925779639839924402495" - , "412596147086332805611392200560087191411541689130482740065137300252639590489" - , "264059866105067484811456793906835462997849523506366903974724979936196358724" - , "80922384260325792825608274004101667366364502159441286540209512108302572137" - , "69261112192907071823876894642934902051254647002357333297635511793652550535" - , "342810644068354896385837929029331085645928473943142618800192452300937098227" - , "228361826362950356801549793202622774869100858404479869989505302905936946659" - , "89244" - ] - , [ "359926778925154809567585559910064420821311221299937568759183366972901588855" - , "128688825421597555299113005875649976344934035310192572516197551102045945994" - , "225379354482053502673695304662016054746658452775856157938886424152134693969" - , "321872319934407904743844034025550739031781361848065513098922085967524180784" - , "250375637325405951645070615947051799520095842815922754899017741501395744611" - , "97676493052788080672307677749501730337551225267342432472194527468634722352" - , "140101187036396881926000630022834247990512766860086619783437252676747730662" - , "428833039353549335335605804240430918631639449874968791377641834408506136850" - , "418359203734539413977740838354554804415161215624809316660001820037711273005" - , "197411877418795659129213175603102238904459737200167987167255995825203749339" - , "221646252100316559257470803343058462853641953756978011126414924636869625612" - , "106393540293584181037890192557883541231531964825708650997196071036779482686" - , "121473330828208543539643554911190528237124877123871673169194452404939833883" - , "234055622144947293638512253368547046093971383516706577723613696162225606040" - , "68307451767502390304445005915787226559811433450721625085741437522802389574" - , "446891883436763112014492564462451523127134145501201571918449345324780633462" - , "83718652783543018019599555197511164121642363321504039439786358267060414949" - , "90267297500929836073049162292104311427365986272517761342871530827272320168" - , "398425606698859520268856768787424690872952910789754531465894080258173664751" - , "323570139379118444589557840594603212198136718240764273769210492735883659788" - , "318597103584099056378057647488068323974418467250708490151864712850204121402" - , "6299083430773359277240726214182464517380839990956316267425262319606033077" - , "27638206326925436960316131682014727983280820447721477666884742925275976240" - , "434344186848869917381375812528446841024676181532946456237439060027443649574" - , "64735754118644738348599761514174981324344130078598038275246522384474432918" - , "53068717269762105498508401788249005564862415051761175636612434108259085043" - , "35813044996911619267309099508360887777226716179396659295580849861836012116" - , "67751791392924142809580984450371772015056060429352159361446914484238646676" - , "68534949135677447506316576616742207938855454921330757052467057435206318183" - , "98510151949547604999069864337574320742530406479752176012935179772005228326" - , "342190252152505345443004241184891966319091967630257822491352072978326623645" - , "362701658859425316334005554473186516818256386066010799465369887406035738447" - , "266999116654850467726292928465517542818678046748008340458185725047959981772" - , "227089355966197874086821090531951502393729872265201602128464978982907992285" - , "240800343500959216904535208047288234867926058830277460630902462914796702354" - , "447956858573680756485556898469710354624642653441041335815079102198306530583" - , "89422712944117481549242421245588048728782658978853365197341587057196539094" - , "72610343179362050463955730204044877712105879926618304878262944723464870506" - , "8676698500519447254981838968537883138182811064381378248657969913325524054" - , "180453700216061196739413267121764366438386031039311941313645439527087166894" - , "63346784016053253727621352882430122335280702556586808389293772066791583857" - , "400031453850139978805133735852120986024101930860924735862305444817383365395" - , "230104622290558570218036071349472289358926019290368625724986905348610140188" - , "175689489221336091369196327293045073133701056385414159213338224521167050830" - , "73310331103509697419315970265031228794034932318600293733074730812482185479" - , "371383255255842707875498538452907102684511927320158672347778293876877893808" - , "165319345890230193939972313953881372394171342391835626454759321320435952720" - , "184753541001210613115361457830691571384268642766106935428207829332011259768" - , "378810733004878432563271790625801205570962762395854282745166380274493181314" - , "86321674336629444862383262780020871828941143514651008200999725989892879714" - , "332634533993388248915777870988529817692938793120418377552247997050250349749" - , "41742010257820712511267405684781534740157292266212120171929853763272599516" - , "224101330592139734390658213442359402546038743346294438455635537496290117560" - , "204363902046137087420878796391135987770418514008394389852388361468850216359" - , "296526036888463156867153847102999031430641220536577556044554029990778763710" - , "137568796227115931047082828377698464185467276723279763304686078869351280509" - , "147456720891843338735232645318045656082153590170441596326500204393398792771" - , "297291342309626392877004635010131510068476699687818509485687346071074942006" - , "20748013593486831233186810444485136836664689496258849465507672301203832324" - , "335431726883875036252568773858744219410188302724423071283643221351691013313" - , "50487384098835523033417523223562560857744547945136829388944024807752630716" - , "425952679139710019732649156938676226492714486376811103362264146658191708598" - , "439787938069461539508728805881194071103269524405653997968488488049426387373" - , "279863410796988495259178322026394289028023166886112128504434877538089779477" - , "398941099058270093463626870965433502581301413791423667994257456160377865247" - , "5759692644185723491187478313102540786562625675495805072053262277490225012" - , "115176" - ] - , [ "199440901482393381753657315848210955092644686957900737971520778206058989647" - , "339215657660349719251187938243949119362753238126598750470832739830379601048" - , "17957417011314891245400567671664723859427624136284133790923936126779445290" - , "294761585889095249928492042608516765584723814657871392207964321318076158536" - , "367304199921887970655898837985855454346911090426896946930048519042744277770" - , "173405546837606747721292792526074597519538685230485266741646923399938591491" - , "13202798104529529703580600642858924379886936325402696094624200032343206719" - , "28211272278315691894282764239983301742024079691520980592618486887749800025" - , "73792448247120972778500624350664849847034095641998271809779791788652649022" - , "386961947078838359430674078072441680475090687247027225632133013772954043342" - , "247859266401821616700765969075270662915024391205665146199401830650793676517" - , "243938047874995926342875119559105623088213951205962677439167259642163766960" - , "14909501249861872673329370269106359532506159818320693170564856401208688898" - , "200331653478898243177761429526240803993101536716611440775588088625522029071" - , "127891684617049394579738365914860024509007913559921966744972525605976847919" - , "202912167983786187592861727924433749852786012202809544200943965898118027816" - , "176370650316309755425558132466370508977563252342126855874617990006444464573" - , "179490319446297562731655155074196383539396893457024237113284509223965454107" - , "118703379899134287650980989454755985628620830085932176414465582081598659194" - , "102025594191113707886629488454876652037341696284939671367279050169222988689" - , "421132375430104331136732058548913808473025321492255503838896123601628815453" - , "328334791815856213206267892694535121226673194052951034497930366807851111845" - , "83012322813281668737061895967682970093636853452224812061110092135287899376" - , "329204708391107220275172926348002826875172581958734129645882445887919889321" - , "410748869385474485539728045765785256294532845070137964737879849265390625591" - , "197274807717335387012872999914232051341799797613667869923402386359379902675" - , "235713095185988155553500217595661312303861624791720350423698435045853678746" - , "150631584359141913552843813384841153102535679219851913018874172414598353488" - , "207783836843813911284913666774420250626019971129676431904589416081127483900" - , "15728034718954665549174230921445077500399069880555428849958014406422697976" - , "69799423545177501667748653663121504155622623013014583690766769624482972893" - , "265665371394145560256710553779588458846778960884758560117805511822299802326" - , "149195925869584039415331014261414953602245337159354350672030141190471260449" - , "162328395279114064180857718453973759378615891406692054752029241989300597156" - , "104643123291849369328362168243087456326274773811561124607060302871149280568" - , "320704123383295724141970902124509237736831907552238934395000039155541269937" - , "77914486216152383968400591704791756847610018757857965758408455442143531631" - , "238365259321088298905088250444146071545398991768186471689605160523129613763" - , "279409375422154593510552116666741774885392805494152031820287626934209978908" - , "195118776021452609708543280509123101181249086555819844047203390788132717252" - , "197977884437087886153482042453896079316138251415359773453987232734849953584" - , "168185043240980087638006965666857387510828226074470344841850764460631595331" - , "231157923359356077977363679818678437836536420121744865399935742538602805912" - , "177903771863742191900138437188329108771172098110036075491750018158192424072" - , "313552174443290416730632310997197097951229162137491595709733420111980331403" - , "273253450712049988786741336540196077743997302924525995219038781650977490211" - , "421908030281055821389377531613150504859996607596444776050212044919345332385" - , "180108184992593746898140529947178182204857361841304042854173884504394805936" - , "37075272799330399065679301151342697855905822543084867570322173216259074746" - , "364885615491975468180698816037289079302391684470748470356247509218051645743" - , "397482868106190800111749908137311511782294652288655656060379563261618687603" - , "192853269627895017416381451198403197013798085262867793715522216458130791820" - , "450480853450142791928572953497280890976328598410525218090104787739409705079" - , "40278654070502961330170439514434669768416784968274956021022493731441898222" - , "251277143131769020481025315216040765839561111684608785317366609258959942695" - , "95094468748825454459610961968601800404132682484160170977941285794444806916" - , "160586633865113902389134480029183924655750088163491531655080014223476604929" - , "211661229493873434581423107377168478957907088187044576032505407590783850232" - , "409651293631434750272174456674594508340952460788494035327119354167465019826" - , "233213211946836553080627522409887799285199986120567245752841080275284294566" - , "143182900674482411759481361336063079267405785923487419697568907351386146653" - , "430050085956999990041799366370428972519385994997821389120583306252090911051" - , "241257468571530133762629460194062384921386438426367660851087853915892684115" - , "106478922860328643074356032194102718325829441005019365153538120054339275205" - , "252933430690486626644000908036895289326526510137514385824014300035301154822" - , "242924628511152862437189415942615812459145003511499320946981326550434266392" - , "107566" - ] - , [ "10892488375325920610152294701785926476935321890453222549428070867493882259" - , "230776541958253414701326765204413805639078570664616139597900921490475143840" - , "162235550819840758141721889536295480113278275911087429090017541814695333320" - , "318634611531007856220026646570492630940047240387334221027051366284225674524" - , "347695480420330337439096561080975864031317110001559084872978387417651465445" - , "243301070227446762125488369714708670219289121859111929877695012393609726208" - , "312153141205681954392441579373652470334482191990502803524039048245142338874" - , "243769659456658813016931656268556893289414122189859136776671772112510762644" - , "235510946617019540983239794402008700611014664578713646813426116406215382253" - , "394638234040056271265534896894991100558052611842099314808878257003754175212" - , "112730195097163222179949980634599934392634120069300673310070655800491242211" - , "112545144551723145227061757291353149296490850338535641681237178578430772288" - , "399161925498018051746424503488168548076537557369822821644005390567188305750" - , "291823556779130095044851238536860577785281148245623338113991004798525195947" - , "443006765181360772964201996695219533539991249361936300010158089981376772939" - , "74018417655448362012716269253153545524112649147848318337218250865231619883" - , "361038295627629757439073080656763633087074408866229383288288831546300069767" - , "269655542872834422597617091710145830035767834433114567250727497135451412216" - , "58289072717559976781527037332370898163550414718655391280446986067842922181" - , "365399954331278626941053447122605263207706937018525782908294459265663426953" - , "83576501872896181822963149907518188809522341045703363171261203985876068484" - , "203403783686919490357886887779316222746544665267595235714814853282937072937" - , "226090172488558641139782927103632452136731207206963592401497570070700221117" - , "249813560776802008219945831355163226098682741553305882699128946352901227282" - , "236586835155013316983057471119105606475813711035522306026673814184519826069" - , "420611449257527132395951061920868895981487081726689195426673433565077012458" - , "414979562418422585161189066520079202660418565693776869373399931253524536378" - , "115851377630895049619958829726730418778383692593973233290077769966938018584" - , "248071158447148977966329235335508229928647083339924308951291087789494073866" - , "8254100651607835318906499132096926759832050649688561036854000785129084907" - , "91385483239630999205401307763890943367451672856478206313711901403403429289" - , "369346641925770698935046191632374874762045839747680407985443471202350286304" - , "236809023553698844817613980395588655518336237009075203833474757804664254158" - , "8367847400805682648908349286699722579431227561083498361702537964306290078" - , "599241730770400067632779114435635342549228985534229813617556932580328166" - , "347112528350917448294202348829052279076907614831011498643025223836596915573" - , "384244379244118003891043669466323943736726794634167201471569326059716944701" - , "118013777197672343498581960057939216208494837962825017767101107204031333144" - , "27234916267695376599463409893017377196853108034589808756909998459364893467" - , "443519198016088819704735929590164254445884637317806160485888215392818578737" - , "396780482611567392375183169345153676737175342167284140440545202776279411157" - , "420351155303051883480975795437743307852799862858964108014000673383502660760" - , "17379377743250873773932440622865094720355292220433235366224143179854831702" - , "299671454782683147939928632170233327590769402224392134648893444626929909373" - , "143062753141414050359792615867774312517100868919516205179025540179759009492" - , "79497692490953838158801094558761984613913564034406069659969793097043605498" - , "422748645389700647011491406944374966856916994331478229959954030359911549565" - , "101802829812014644970197499895811874607753186302439171072935333706660468030" - , "376428369998893026519415315112012919906032811618495880392785036762185101192" - , "193969030999254195249242252871597931610859408264053152789041067245597391073" - , "262277607928686742238285487190873200602833495734085188071246746209841324139" - , "154099884960502807271641574310268486840763221700920893692135020347157046386" - , "155875061164585018658671842995328931296342883770473498362059838106632382461" - , "248574435283666782825705601259695525637993294311364397935499480206725256362" - , "171325185063038052248966557755722232979612702743265316145563443527135798688" - , "19982746887818897250405980185937061235439217109294376948752373205830077881" - , "363719103724181291346130833008745948141602173373912337315631878022251200824" - , "174596812883797666354579747966720458118607233660798323559531788300018084931" - , "296611197821867585469311917529661698312828606304722408477045992233526328708" - , "115884038550627840768260751168697204665962542002024023842649383174518895165" - , "265597417366164841889730505737916646261040505851159477903649773521314216810" - , "59890857222664166616144499778264545115438678877460924559608721809777680238" - , "150275344313515259978222149421664752546204516114655782705875535407472455999" - , "119762211657733951640135703777940013374803447228667976561992857374027112851" - , "124750313254270944205036764607428674226393862430770588148329786501973600535" - , "223562415856611692667255745278751292230311455355383541649772348032933666931" - , "70851" - ] - ] -, "merklePaths": - [ [ "12330511756602656312909435206144412037562550923861053314147193494605624608532" - , "11626412651279744307149389430094868876817959605147995292836995419396445628874" - , "5992799448428980485292103987999399446160713735809250167288027256759377161164" - , "19665782623633007009046423286205585606029554995684266199914536255425604862856" - , "16487082247902739801276638140231448150060608377815926330854457096549924699346" - , "13757776896542890425183206586238760231244468647794123671233758868377423038254" - , "5689382212199289790852870716648320118282913659439556386010574440270030991956" - , "19397819444005071538172083272808000152189643623980212079104170950073560541073" - , "13602141253349313166682170066159161155345481788270338632198566205283140117430" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - ] - , [ "20475873274412005481064639175076772466269555202404881992985142339142752174247" - , "16346160910918020455716115524174351294558871582658635761456705803199256575588" - , "2853750013041818757293788273269978233503065226276819540991640203844566736443" - , "9192572535522846104757923561847416578417599996904208474460700268961782558170" - , "11041850361074018956732434352769265303294549935255362322653487210796196161858" - , "20835509643844784930831626622202658364476409300598072395494952478408974334325" - , "15426115581767819720710837762133134950520914636073261355445708100826108573907" - , "7565353224987902191368863653499353764559862741092477570130316358454603122676" - , "2622681935585012630617774892501744551457568716225188460692779556142778732663" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - ] - , [ "13099868869639061574800067722436547911616384523753701384396275064543709640456" - , "353757809120595355213201328586632712724405232919181040928026587840976194078" - , "17653300914565730132855106316678548541847283141888106932466281385199556950861" - , "15467462085462582082877261755656498905479817107855355753427299990712166382496" - , "8291733777946446853018893495264026584437749231931118771866890345692346711355" - , "15510790697317206014779022286261864844918915222875014882833700758879700055506" - , "5689382212199289790852870716648320118282913659439556386010574440270030991956" - , "19397819444005071538172083272808000152189643623980212079104170950073560541073" - , "13602141253349313166682170066159161155345481788270338632198566205283140117430" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - ] - , [ "123238869181525326412236116167249816873084559218151119452851092131080991962" - , "9610314342084317647296061595824603740114670828357751076517430572434680540425" - , "16802740554584732104294972716558962567961331277692246600846665155168171370476" - , "151083360419914122898584757765086723506432610661508069194962432698872036623" - , "10357032992337239725601662829902169825217513617307319193581711776597892496381" - , "10120699018002766520605012835043517238241846918467244955580419060582311503402" - , "21149604008153751948441881526949680605328007895979738537313721955134548786062" - , "5720106921878932614189421948890362637585879521377362100104826996201092964473" - , "2622681935585012630617774892501744551457568716225188460692779556142778732663" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - ] - , [ "19153513112714782931111012244694922243101677748840856395814929006033044311081" - , "21046138187228318287277629107063936540039891592394801899272249280765102572688" - , "18057980437430910028015917806534512217725128031222973066601095455076586015436" - , "5766677914654397407881589917461473873246279171605373166264025525757502238061" - , "12019967669236656188577515900815533059046454955207846938479617973037184411021" - , "14504305765289705714959523666100275156034056689367568164630385862257567596209" - , "7152002871325824138073253423783370852632926621899161541618248808716037342022" - , "9714587356194206699401761190093056901650105401919163689816999407566849779455" - , "13602141253349313166682170066159161155345481788270338632198566205283140117430" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - , "0" - ] - ] -} diff --git a/tests/circuits/fixtures/proof_main.r1cs b/tests/circuits/fixtures/proof_main.r1cs deleted file mode 100644 index 8b58ffa8..00000000 Binary files a/tests/circuits/fixtures/proof_main.r1cs and /dev/null differ diff --git a/tests/circuits/fixtures/proof_main.wasm b/tests/circuits/fixtures/proof_main.wasm deleted file mode 100644 index f908d4f7..00000000 Binary files a/tests/circuits/fixtures/proof_main.wasm and /dev/null differ diff --git a/tests/circuits/fixtures/proof_main.zkey b/tests/circuits/fixtures/proof_main.zkey deleted file mode 100644 index 5451ad0f..00000000 Binary files a/tests/circuits/fixtures/proof_main.zkey and /dev/null differ diff --git a/tests/codex/blockexchange/discovery/testdiscovery.nim b/tests/codex/blockexchange/discovery/testdiscovery.nim index 6b36a630..b828d57e 100644 --- a/tests/codex/blockexchange/discovery/testdiscovery.nim +++ b/tests/codex/blockexchange/discovery/testdiscovery.nim @@ -32,7 +32,6 @@ asyncchecksuite "Block Advertising and Discovery": blockDiscovery: MockDiscovery discovery: DiscoveryEngine advertiser: Advertiser - wallet: WalletRef network: BlockExcNetwork localStore: CacheStore engine: BlockExcEngine @@ -48,7 +47,6 @@ asyncchecksuite "Block Advertising and Discovery": switch = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) blockDiscovery = MockDiscovery.new() - wallet = WalletRef.example network = BlockExcNetwork.new(switch) localStore = CacheStore.new(blocks.mapIt(it)) peerStore = PeerCtxStore.new() @@ -72,7 +70,7 @@ asyncchecksuite "Block Advertising and Discovery": advertiser = Advertiser.new(localStore, blockDiscovery) engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + localStore, network, discovery, advertiser, peerStore, pendingBlocks ) switch.mount(network) @@ -132,7 +130,7 @@ asyncchecksuite "Block Advertising and Discovery": peerId = PeerId.example haves = collect(initTable()): for blk in blocks: - {blk.address: Presence(address: blk.address, price: 0.u256)} + {blk.address: Presence(address: blk.address)} engine.peers.add(BlockExcPeerCtx(id: peerId, blocks: haves)) @@ -180,7 +178,6 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": let s = newStandardSwitch(transportFlags = {ServerFlags.ReuseAddr}) blockDiscovery = MockDiscovery.new() - wallet = WalletRef.example network = BlockExcNetwork.new(s) localStore = CacheStore.new() peerStore = PeerCtxStore.new() @@ -198,7 +195,7 @@ asyncchecksuite "E2E - Multiple Nodes Discovery": advertiser = Advertiser.new(localStore, blockDiscovery) engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + localStore, network, discovery, advertiser, peerStore, pendingBlocks ) networkStore = NetworkStore.new(engine, localStore) diff --git a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim index df3f8c80..b7282a92 100644 --- a/tests/codex/blockexchange/discovery/testdiscoveryengine.nim +++ b/tests/codex/blockexchange/discovery/testdiscoveryengine.nim @@ -130,7 +130,7 @@ asyncchecksuite "Test Discovery Engine": let address = BlockAddress(leaf: false, cid: cid) - peerCtx.blocks[address] = Presence(address: address, price: 0.u256) + peerCtx.blocks[address] = Presence(address: address) peerStore.add(peerCtx) want.fire() diff --git a/tests/codex/blockexchange/engine/testblockexc.nim b/tests/codex/blockexchange/engine/testblockexc.nim index 7c2a9ed8..1ac035dd 100644 --- a/tests/codex/blockexchange/engine/testblockexc.nim +++ b/tests/codex/blockexchange/engine/testblockexc.nim @@ -19,7 +19,6 @@ asyncchecksuite "NetworkStore engine - 2 nodes": var nodeCmps1, nodeCmps2: NodesComponents peerCtx1, peerCtx2: BlockExcPeerCtx - pricing1, pricing2: Pricing blocks1, blocks2: seq[bt.Block] pendingBlocks1, pendingBlocks2: seq[BlockHandle] @@ -38,14 +37,6 @@ asyncchecksuite "NetworkStore engine - 2 nodes": pendingBlocks2 = blocks1[0 .. 3].mapIt(nodeCmps2.pendingBlocks.getWantHandle(it.cid)) - pricing1 = Pricing.example() - pricing2 = Pricing.example() - - pricing1.address = nodeCmps1.wallet.address - pricing2.address = nodeCmps2.wallet.address - nodeCmps1.engine.pricing = pricing1.some - nodeCmps2.engine.pricing = pricing2.some - await nodeCmps1.switch.connect( nodeCmps2.switch.peerInfo.peerId, nodeCmps2.switch.peerInfo.addrs ) @@ -75,10 +66,6 @@ asyncchecksuite "NetworkStore engine - 2 nodes": .mapIt($it.read.get.cid) .sorted(cmp[string]) == blocks2[0 .. 3].mapIt($it.cid).sorted(cmp[string]) - test "Should exchanges accounts on connect": - check peerCtx1.account .? address == pricing1.address.some - check peerCtx2.account .? address == pricing2.address.some - test "Should send want-have for block": let blk = bt.Block.new("Block 1".toBytes).tryGet() let blkFut = nodeCmps1.pendingBlocks.getWantHandle(blk.cid) @@ -112,19 +99,6 @@ asyncchecksuite "NetworkStore engine - 2 nodes": check await nodeCmps1.networkStore.getBlock(blk.cid).withTimeout(100.millis) # should succeed - test "Should receive payments for blocks that were sent": - discard - await allFinished(blocks2[4 .. 7].mapIt(nodeCmps2.networkStore.putBlock(it))) - - discard - await allFinished(blocks2[4 .. 7].mapIt(nodeCmps1.networkStore.getBlock(it.cid))) - - let - channel = !peerCtx1.paymentChannel - wallet = nodeCmps2.wallet - - check eventually wallet.balance(channel, Asset) > 0 - asyncchecksuite "NetworkStore - multiple nodes": var nodes: seq[NodesComponents] diff --git a/tests/codex/blockexchange/engine/testengine.nim b/tests/codex/blockexchange/engine/testengine.nim index 1afe2147..07573cf2 100644 --- a/tests/codex/blockexchange/engine/testengine.nim +++ b/tests/codex/blockexchange/engine/testengine.nim @@ -29,7 +29,6 @@ asyncchecksuite "NetworkStore engine basic": var peerId: PeerId chunker: Chunker - wallet: WalletRef blockDiscovery: Discovery peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager @@ -39,7 +38,6 @@ asyncchecksuite "NetworkStore engine basic": setup: peerId = PeerId.example chunker = RandomChunker.new(Rng.instance(), size = 1024'nb, chunkSize = 256'nb) - wallet = WalletRef.example blockDiscovery = Discovery.new() peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() @@ -74,7 +72,7 @@ asyncchecksuite "NetworkStore engine basic": ) advertiser = Advertiser.new(localStore, blockDiscovery) engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + localStore, network, discovery, advertiser, peerStore, pendingBlocks ) for b in blocks: @@ -83,39 +81,10 @@ asyncchecksuite "NetworkStore engine basic": await done.wait(100.millis) - test "Should send account to new peers": - let pricing = Pricing.example - - proc sendAccount( - peer: PeerId, account: Account - ) {.async: (raises: [CancelledError]).} = - check account.address == pricing.address - done.complete() - - let - network = BlockExcNetwork(request: BlockExcRequest(sendAccount: sendAccount)) - - localStore = CacheStore.new() - discovery = DiscoveryEngine.new( - localStore, peerStore, network, blockDiscovery, pendingBlocks - ) - - advertiser = Advertiser.new(localStore, blockDiscovery) - - engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks - ) - - engine.pricing = pricing.some - await engine.peerAddedHandler(peerId) - - await done.wait(100.millis) - asyncchecksuite "NetworkStore engine handlers": var peerId: PeerId chunker: Chunker - wallet: WalletRef blockDiscovery: Discovery peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager @@ -138,7 +107,6 @@ asyncchecksuite "NetworkStore engine handlers": blocks.add(Block.new(chunk).tryGet()) peerId = PeerId.example - wallet = WalletRef.example blockDiscovery = Discovery.new() peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() @@ -152,7 +120,7 @@ asyncchecksuite "NetworkStore engine handlers": advertiser = Advertiser.new(localStore, blockDiscovery) engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + localStore, network, discovery, advertiser, peerStore, pendingBlocks ) peerCtx = BlockExcPeerCtx(id: peerId) @@ -258,46 +226,6 @@ asyncchecksuite "NetworkStore engine handlers": let present = await engine.localStore.hasBlock(b.cid) check present.tryGet() - test "Should send payments for received blocks": - let - done = newFuture[void]() - account = Account(address: EthAddress.example) - peerContext = peerStore.get(peerId) - - peerContext.account = account.some - peerContext.blocks = blocks.mapIt( - (it.address, Presence(address: it.address, price: rand(uint16).u256, have: true)) - ).toTable - - for blk in blocks: - peerContext.blockRequestScheduled(blk.address) - - engine.network = BlockExcNetwork( - request: BlockExcRequest( - sendPayment: proc( - receiver: PeerId, payment: SignedState - ) {.async: (raises: [CancelledError]).} = - let - amount = - blocks.mapIt(peerContext.blocks[it.address].catch.get.price).foldl(a + b) - balances = !payment.state.outcome.balances(Asset) - - check receiver == peerId - check balances[account.address.toDestination].catch.get == amount - done.complete(), - - # Install NOP for want list cancellations so they don't cause a crash - sendWantCancellations: NopSendWantCancellationsProc, - ) - ) - - let requestedBlocks = blocks.mapIt(engine.pendingBlocks.getWantHandle(it.address)) - await engine.blocksDeliveryHandler( - peerId, blocks.mapIt(BlockDelivery(blk: it, address: it.address)) - ) - await done.wait(100.millis) - await allFuturesThrowing(requestedBlocks).wait(100.millis) - test "Should handle block presence": var handles: Table[Cid, Future[Block].Raising([CancelledError, RetriesExhaustedError])] @@ -323,17 +251,15 @@ asyncchecksuite "NetworkStore engine handlers": # only Cids in peer want lists are requested handles = blocks.mapIt((it.cid, engine.pendingBlocks.getWantHandle(it.cid))).toTable - let price = UInt256.example await engine.blockPresenceHandler( peerId, blocks.mapIt( - PresenceMessage.init(Presence(address: it.address, have: true, price: price)) + PresenceMessage.init(Presence(address: it.address, have: true)) ), ) for a in blocks.mapIt(it.address): check a in peerCtx.peerHave - check peerCtx.blocks[a].price == price test "Should send cancellations for requested blocks only": let @@ -376,7 +302,6 @@ asyncchecksuite "Block Download": seckey: PrivateKey peerId: PeerId chunker: Chunker - wallet: WalletRef blockDiscovery: Discovery peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager @@ -399,7 +324,6 @@ asyncchecksuite "Block Download": blocks.add(Block.new(chunk).tryGet()) peerId = PeerId.example - wallet = WalletRef.example blockDiscovery = Discovery.new() peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() @@ -413,7 +337,7 @@ asyncchecksuite "Block Download": advertiser = Advertiser.new(localStore, blockDiscovery) engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + localStore, network, discovery, advertiser, peerStore, pendingBlocks ) peerCtx = BlockExcPeerCtx(id: peerId, activityTimeout: 100.milliseconds) @@ -540,7 +464,6 @@ asyncchecksuite "Task Handler": var peerId: PeerId chunker: Chunker - wallet: WalletRef blockDiscovery: Discovery peerStore: PeerCtxStore pendingBlocks: PendingBlocksManager @@ -564,7 +487,6 @@ asyncchecksuite "Task Handler": blocks.add(Block.new(chunk).tryGet()) peerId = PeerId.example - wallet = WalletRef.example blockDiscovery = Discovery.new() peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() @@ -578,7 +500,7 @@ asyncchecksuite "Task Handler": advertiser = Advertiser.new(localStore, blockDiscovery) engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + localStore, network, discovery, advertiser, peerStore, pendingBlocks ) peersCtx = @[] @@ -587,8 +509,6 @@ asyncchecksuite "Task Handler": peersCtx.add(BlockExcPeerCtx(id: peers[i])) peerStore.add(peersCtx[i]) - engine.pricing = Pricing.example.some - # FIXME: this is disabled for now: I've dropped block priorities to make # my life easier as I try to optimize the protocol, and also because # they were not being used anywhere. diff --git a/tests/codex/blockexchange/engine/testpayments.nim b/tests/codex/blockexchange/engine/testpayments.nim deleted file mode 100644 index e93cc837..00000000 --- a/tests/codex/blockexchange/engine/testpayments.nim +++ /dev/null @@ -1,33 +0,0 @@ -import pkg/unittest2 - -import pkg/codex/stores -import ../../examples -import ../../helpers - -suite "Engine payments": - let address = EthAddress.example - let amount = 42.u256 - - var wallet: WalletRef - var peer: BlockExcPeerCtx - - setup: - wallet = WalletRef.example - peer = BlockExcPeerCtx.example - peer.account = Account(address: address).some - - test "pays for received blocks": - let payment = !wallet.pay(peer, amount) - let balances = payment.state.outcome.balances(Asset) - let destination = address.toDestination - check !balances[destination] == amount - - test "no payment when no account is set": - peer.account = Account.none - check wallet.pay(peer, amount).isFailure - - test "uses same channel for consecutive payments": - let payment1, payment2 = wallet.pay(peer, amount) - let channel1 = payment1 .? state .? channel .? getChannelId - let channel2 = payment2 .? state .? channel .? getChannelId - check channel1 == channel2 diff --git a/tests/codex/blockexchange/protobuf/testpayments.nim b/tests/codex/blockexchange/protobuf/testpayments.nim deleted file mode 100644 index 3ada0105..00000000 --- a/tests/codex/blockexchange/protobuf/testpayments.nim +++ /dev/null @@ -1,37 +0,0 @@ -import pkg/chronos -import pkg/stew/byteutils -import pkg/codex/stores - -import ../../../asynctest -import ../../examples -import ../../helpers - -suite "account protobuf messages": - let account = Account(address: EthAddress.example) - let message = AccountMessage.init(account) - - test "encodes recipient of payments": - check message.address == @(account.address.toArray) - - test "decodes recipient of payments": - check Account.init(message) .? address == account.address.some - - test "fails to decode when address has incorrect number of bytes": - var incorrect = message - incorrect.address.del(0) - check Account.init(incorrect).isNone - -suite "channel update messages": - let state = SignedState.example - let update = StateChannelUpdate.init(state) - - test "encodes a nitro signed state": - check update.update == state.toJson.toBytes - - test "decodes a channel update": - check SignedState.init(update) == state.some - - test "fails to decode incorrect channel update": - var incorrect = update - incorrect.update.del(0) - check SignedState.init(incorrect).isNone diff --git a/tests/codex/blockexchange/protobuf/testpresence.nim b/tests/codex/blockexchange/protobuf/testpresence.nim index dc048c59..2f449e15 100644 --- a/tests/codex/blockexchange/protobuf/testpresence.nim +++ b/tests/codex/blockexchange/protobuf/testpresence.nim @@ -10,8 +10,7 @@ suite "block presence protobuf messages": let cid = Cid.example address = BlockAddress(leaf: false, cid: cid) - price = UInt256.example - presence = Presence(address: address, have: true, price: price) + presence = Presence(address: address, have: true) message = PresenceMessage.init(presence) test "encodes have/donthave": @@ -21,9 +20,6 @@ suite "block presence protobuf messages": presence.have = false check PresenceMessage.init(presence).`type` == DontHave - test "encodes price": - check message.price == @(price.toBytesBE) - test "decodes CID": check Presence.init(message) .? address == address.some @@ -33,11 +29,3 @@ suite "block presence protobuf messages": check Presence.init(message) .? have == true.some message.`type` = BlockPresenceType.DontHave check Presence.init(message) .? have == false.some - - test "decodes price": - check Presence.init(message) .? price == price.some - - test "fails to decode when price is invalid": - var incorrect = message - incorrect.price.add(0) - check Presence.init(incorrect).isNone diff --git a/tests/codex/blockexchange/testengine.nim b/tests/codex/blockexchange/testengine.nim index 9cd968ee..cb5f91ae 100644 --- a/tests/codex/blockexchange/testengine.nim +++ b/tests/codex/blockexchange/testengine.nim @@ -1,6 +1,5 @@ import ./engine/testengine import ./engine/testblockexc -import ./engine/testpayments import ./engine/testadvertiser {.warning[UnusedImport]: off.} diff --git a/tests/codex/blockexchange/testnetwork.nim b/tests/codex/blockexchange/testnetwork.nim index ab19a6ae..29646bc5 100644 --- a/tests/codex/blockexchange/testnetwork.nim +++ b/tests/codex/blockexchange/testnetwork.nim @@ -104,34 +104,6 @@ asyncchecksuite "Network - Handlers": await done.wait(500.millis) - test "Handles account messages": - let account = Account(address: EthAddress.example) - - proc handleAccount(peer: PeerId, received: Account) {.async: (raises: []).} = - check received == account - done.complete() - - network.handlers.onAccount = handleAccount - - let message = Message(account: AccountMessage.init(account)) - await buffer.pushData(lenPrefix(protobufEncode(message))) - - await done.wait(100.millis) - - test "Handles payment messages": - let payment = SignedState.example - - proc handlePayment(peer: PeerId, received: SignedState) {.async: (raises: []).} = - check received == payment - done.complete() - - network.handlers.onPayment = handlePayment - - let message = Message(payment: StateChannelUpdate.init(payment)) - await buffer.pushData(lenPrefix(protobufEncode(message))) - - await done.wait(100.millis) - asyncchecksuite "Network - Senders": let chunker = RandomChunker.new(Rng.instance(), size = 1024, chunkSize = 256) @@ -227,30 +199,6 @@ asyncchecksuite "Network - Senders": await done.wait(500.millis) - test "send account": - let account = Account(address: EthAddress.example) - - proc handleAccount(peer: PeerId, received: Account) {.async: (raises: []).} = - check received == account - done.complete() - - network2.handlers.onAccount = handleAccount - - await network1.sendAccount(switch2.peerInfo.peerId, account) - await done.wait(500.millis) - - test "send payment": - let payment = SignedState.example - - proc handlePayment(peer: PeerId, received: SignedState) {.async: (raises: []).} = - check received == payment - done.complete() - - network2.handlers.onPayment = handlePayment - - await network1.sendPayment(switch2.peerInfo.peerId, payment) - await done.wait(500.millis) - asyncchecksuite "Network - Test Limits": var switch1, switch2: Switch @@ -277,14 +225,14 @@ asyncchecksuite "Network - Test Limits": await allFuturesThrowing(switch1.stop(), switch2.stop()) test "Concurrent Sends": - let account = Account(address: EthAddress.example) - network2.handlers.onAccount = proc( - peer: PeerId, received: Account - ) {.async: (raises: []).} = + network2.handlers.onPresence = proc( + peer: PeerId, presence: seq[BlockPresence] + ): Future[void] {.async: (raises: []).} = check false let fut = network1.send( - switch2.peerInfo.peerId, Message(account: AccountMessage.init(account)) + switch2.peerInfo.peerId, + Message() ) await sleepAsync(100.millis) diff --git a/tests/codex/blockexchange/testpeerctxstore.nim b/tests/codex/blockexchange/testpeerctxstore.nim index f348c1d5..880e08c3 100644 --- a/tests/codex/blockexchange/testpeerctxstore.nim +++ b/tests/codex/blockexchange/testpeerctxstore.nim @@ -58,11 +58,11 @@ suite "Peer Context Store Peer Selection": test "Should select peers that have Cid": peerCtxs[0].blocks = collect(initTable): for i, a in addresses: - {a: Presence(address: a, price: i.u256)} + {a: Presence(address: a)} peerCtxs[5].blocks = collect(initTable): for i, a in addresses: - {a: Presence(address: a, price: i.u256)} + {a: Presence(address: a)} let peers = store.peersHave(addresses[0]) @@ -94,8 +94,8 @@ suite "Peer Context Store Peer Selection": test "Should return peers with and without block": let address = addresses[2] - peerCtxs[1].blocks[address] = Presence(address: address, price: 0.u256) - peerCtxs[2].blocks[address] = Presence(address: address, price: 0.u256) + peerCtxs[1].blocks[address] = Presence(address: address) + peerCtxs[2].blocks[address] = Presence(address: address) let peers = store.getPeersForBlock(address) diff --git a/tests/codex/blockexchange/testprotobuf.nim b/tests/codex/blockexchange/testprotobuf.nim index 1d41d6b4..9f185cfe 100644 --- a/tests/codex/blockexchange/testprotobuf.nim +++ b/tests/codex/blockexchange/testprotobuf.nim @@ -1,4 +1,3 @@ -import ./protobuf/testpayments import ./protobuf/testpresence {.warning[UnusedImport]: off.} diff --git a/tests/codex/examples.nim b/tests/codex/examples.nim index 260adbfc..4c9535cb 100644 --- a/tests/codex/examples.nim +++ b/tests/codex/examples.nim @@ -1,42 +1,16 @@ import std/random import std/sequtils import pkg/libp2p -import pkg/nitro import pkg/stint import pkg/codex/rng import pkg/codex/stores import pkg/codex/blocktype as bt -import pkg/codex/sales import pkg/codex/merkletree import pkg/codex/manifest import ../examples export examples -proc example*(_: type EthAddress): EthAddress = - EthPrivateKey.random().toPublicKey.toAddress - -proc example*(_: type UInt48): UInt48 = - # workaround for https://github.com/nim-lang/Nim/issues/17670 - uint64.rand mod (UInt48.high + 1) - -proc example*(_: type Wallet): Wallet = - Wallet.init(EthPrivateKey.random()) - -proc example*(_: type WalletRef): WalletRef = - WalletRef.new(EthPrivateKey.random()) - -proc example*(_: type SignedState): SignedState = - var wallet = Wallet.example - let hub, asset, receiver = EthAddress.example - let chainId, amount = UInt256.example - let nonce = UInt48.example - let channel = wallet.openLedgerChannel(hub, chainId, nonce, asset, amount).get - wallet.pay(channel, asset, receiver, amount).get - -proc example*(_: type Pricing): Pricing = - Pricing(address: EthAddress.example, price: uint32.rand.u256) - proc example*(_: type bt.Block, size: int = 4096): bt.Block = let bytes = newSeqWith(size, rand(uint8)) bt.Block.new(bytes).tryGet() @@ -51,6 +25,10 @@ proc example*(_: type BlockExcPeerCtx): BlockExcPeerCtx = proc example*(_: type Cid): Cid = bt.Block.example.cid +proc example*(_: type BlockAddress): BlockAddress = + let cid = Cid.example + BlockAddress.init(cid) + proc example*(_: type Manifest): Manifest = Manifest.new( treeCid = Cid.example, @@ -64,32 +42,5 @@ proc example*(_: type MultiHash, mcodec = Sha256HashCodec): MultiHash = let bytes = newSeqWith(256, rand(uint8)) MultiHash.digest($mcodec, bytes).tryGet() -proc example*( - _: type Availability, collateralPerByte = uint8.example.u256 -): Availability = - let totalSize = uint16.example.uint64 - Availability.init( - totalSize = totalSize, - freeSize = uint16.example.uint64, - duration = uint16.example.uint64, - minPricePerBytePerSecond = uint8.example.u256, - totalCollateral = totalSize.u256 * collateralPerByte, - enabled = true, - until = 0.SecondsSince1970, - ) - -proc example*(_: type Reservation): Reservation = - Reservation.init( - availabilityId = AvailabilityId(array[32, byte].example), - size = uint16.example.uint64, - slotId = SlotId.example, - ) - proc example*(_: type MerkleProof): MerkleProof = MerkleProof.init(3, @[MultiHash.example]).tryget() - -proc example*(_: type Poseidon2Proof): Poseidon2Proof = - var example = MerkleProof[Poseidon2Hash, PoseidonKeysEnum]() - example.index = 123 - example.path = @[1, 2, 3, 4].mapIt(it.toF) - example diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim deleted file mode 100644 index 4483a0d6..00000000 --- a/tests/codex/helpers/mockmarket.nim +++ /dev/null @@ -1,646 +0,0 @@ -import std/sequtils -import std/tables -import std/hashes -import std/sets -import std/sugar -import pkg/questionable -import pkg/codex/market -import pkg/codex/contracts/requests -import pkg/codex/contracts/proofs -import pkg/codex/contracts/config -import pkg/questionable/results - -from pkg/ethers import BlockTag -import codex/clock - -import ../examples -import ./mockclock - -export market -export tables - -logScope: - topics = "mockMarket" - -type - MockMarket* = ref object of Market - periodicity: Periodicity - activeRequests*: Table[Address, seq[RequestId]] - activeSlots*: Table[Address, seq[SlotId]] - requested*: seq[StorageRequest] - requestEnds*: Table[RequestId, SecondsSince1970] - requestExpiry*: Table[RequestId, SecondsSince1970] - requestState*: Table[RequestId, RequestState] - slotState*: Table[SlotId, SlotState] - fulfilled*: seq[Fulfillment] - filled*: seq[MockSlot] - freed*: seq[SlotId] - submitted*: seq[Groth16Proof] - markedAsMissingProofs*: seq[SlotId] - canBeMarkedAsMissing: HashSet[SlotId] - withdrawn*: seq[RequestId] - proofPointer*: uint8 - proofsRequired: HashSet[SlotId] - proofsToBeRequired: HashSet[SlotId] - proofChallenge*: ProofChallenge - proofEnds: Table[SlotId, UInt256] - signer: Address - subscriptions: Subscriptions - config*: MarketplaceConfig - canReserveSlot*: bool - errorOnReserveSlot*: ?(ref MarketError) - errorOnFillSlot*: ?(ref MarketError) - errorOnFreeSlot*: ?(ref MarketError) - errorOnGetHost*: ?(ref MarketError) - clock: Clock - - Fulfillment* = object - requestId*: RequestId - proof*: Groth16Proof - host*: Address - - MockSlot* = object - requestId*: RequestId - host*: Address - slotIndex*: uint64 - proof*: Groth16Proof - timestamp: SecondsSince1970 - collateral*: UInt256 - - Subscriptions = object - onRequest: seq[RequestSubscription] - onFulfillment: seq[FulfillmentSubscription] - onSlotFilled: seq[SlotFilledSubscription] - onSlotFreed: seq[SlotFreedSubscription] - onSlotReservationsFull: seq[SlotReservationsFullSubscription] - onRequestCancelled: seq[RequestCancelledSubscription] - onRequestFailed: seq[RequestFailedSubscription] - onProofSubmitted: seq[ProofSubmittedSubscription] - - RequestSubscription* = ref object of Subscription - market: MockMarket - callback: OnRequest - - FulfillmentSubscription* = ref object of Subscription - market: MockMarket - requestId: ?RequestId - callback: OnFulfillment - - SlotFilledSubscription* = ref object of Subscription - market: MockMarket - requestId: ?RequestId - slotIndex: ?uint64 - callback: OnSlotFilled - - SlotFreedSubscription* = ref object of Subscription - market: MockMarket - callback: OnSlotFreed - - SlotReservationsFullSubscription* = ref object of Subscription - market: MockMarket - callback: OnSlotReservationsFull - - RequestCancelledSubscription* = ref object of Subscription - market: MockMarket - requestId: ?RequestId - callback: OnRequestCancelled - - RequestFailedSubscription* = ref object of Subscription - market: MockMarket - requestId: ?RequestId - callback: OnRequestCancelled - - ProofSubmittedSubscription = ref object of Subscription - market: MockMarket - callback: OnProofSubmitted - -proc hash*(address: Address): Hash = - hash(address.toArray) - -proc hash*(requestId: RequestId): Hash = - hash(requestId.toArray) - -proc new*(_: type MockMarket, clock: Clock = MockClock.new()): MockMarket = - ## Create a new mocked Market instance - ## - let config = MarketplaceConfig( - collateral: CollateralConfig( - repairRewardPercentage: 10, - maxNumberOfSlashes: 5, - slashPercentage: 10, - validatorRewardPercentage: 20, - ), - proofs: ProofConfig( - period: 10.Period, - timeout: 5.uint64, - downtime: 64.uint8, - downtimeProduct: 67.uint8, - ), - reservations: SlotReservationsConfig(maxReservations: 3), - requestDurationLimit: (60 * 60 * 24 * 30).uint64, - ) - MockMarket( - signer: Address.example, config: config, canReserveSlot: true, clock: clock - ) - -method loadConfig*( - market: MockMarket -): Future[?!void] {.async: (raises: [CancelledError]).} = - discard - -method getSigner*( - market: MockMarket -): Future[Address] {.async: (raises: [CancelledError, MarketError]).} = - return market.signer - -method periodicity*( - mock: MockMarket -): Future[Periodicity] {.async: (raises: [CancelledError, MarketError]).} = - return Periodicity(seconds: mock.config.proofs.period) - -method proofTimeout*( - market: MockMarket -): Future[uint64] {.async: (raises: [CancelledError, MarketError]).} = - return market.config.proofs.timeout - -method requestDurationLimit*(market: MockMarket): Future[uint64] {.async.} = - return market.config.requestDurationLimit - -method proofDowntime*( - market: MockMarket -): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} = - return market.config.proofs.downtime - -method repairRewardPercentage*( - market: MockMarket -): Future[uint8] {.async: (raises: [CancelledError, MarketError]).} = - return market.config.collateral.repairRewardPercentage - -method getPointer*(market: MockMarket, slotId: SlotId): Future[uint8] {.async.} = - return market.proofPointer - -method requestStorage*( - market: MockMarket, request: StorageRequest -) {.async: (raises: [CancelledError, MarketError]).} = - let now = market.clock.now() - let requestExpiresAt = now + request.expiry.toSecondsSince1970 - let requestEndsAt = now + request.ask.duration.toSecondsSince1970 - market.requested.add(request) - market.requestExpiry[request.id] = requestExpiresAt - market.requestEnds[request.id] = requestEndsAt - var subscriptions = market.subscriptions.onRequest - for subscription in subscriptions: - subscription.callback(request.id, request.ask, requestExpiresAt.uint64) - -method myRequests*(market: MockMarket): Future[seq[RequestId]] {.async.} = - return market.activeRequests[market.signer] - -method mySlots*(market: MockMarket): Future[seq[SlotId]] {.async.} = - return market.activeSlots[market.signer] - -method getRequest*( - market: MockMarket, id: RequestId -): Future[?StorageRequest] {.async: (raises: [CancelledError]).} = - for request in market.requested: - if request.id == id: - return some request - return none StorageRequest - -method getActiveSlot*(market: MockMarket, id: SlotId): Future[?Slot] {.async.} = - for slot in market.filled: - if slotId(slot.requestId, slot.slotIndex) == id and - request =? await market.getRequest(slot.requestId): - return some Slot(request: request, slotIndex: slot.slotIndex) - return none Slot - -method requestState*( - market: MockMarket, requestId: RequestId -): Future[?RequestState] {.async.} = - return market.requestState .? [requestId] - -method slotState*( - market: MockMarket, slotId: SlotId -): Future[SlotState] {.async: (raises: [CancelledError, MarketError]).} = - if slotId notin market.slotState: - return SlotState.Free - - try: - return market.slotState[slotId] - except KeyError as e: - raiseAssert "SlotId not found in known slots (MockMarket.slotState)" - -method getRequestEnd*( - market: MockMarket, id: RequestId -): Future[SecondsSince1970] {.async.} = - return market.requestEnds[id] - -method requestExpiresAt*( - market: MockMarket, id: RequestId -): Future[SecondsSince1970] {.async.} = - return market.requestExpiry[id] - -method getHost*( - market: MockMarket, requestId: RequestId, slotIndex: uint64 -): Future[?Address] {.async: (raises: [CancelledError, MarketError]).} = - if error =? market.errorOnGetHost: - raise error - - for slot in market.filled: - if slot.requestId == requestId and slot.slotIndex == slotIndex: - return some slot.host - return none Address - -method currentCollateral*( - market: MockMarket, slotId: SlotId -): Future[UInt256] {.async: (raises: [MarketError, CancelledError]).} = - for slot in market.filled: - if slotId == slotId(slot.requestId, slot.slotIndex): - return slot.collateral - return 0.u256 - -proc emitSlotFilled*(market: MockMarket, requestId: RequestId, slotIndex: uint64) = - var subscriptions = market.subscriptions.onSlotFilled - for subscription in subscriptions: - let requestMatches = - subscription.requestId.isNone or subscription.requestId == some requestId - let slotMatches = - subscription.slotIndex.isNone or subscription.slotIndex == some slotIndex - if requestMatches and slotMatches: - subscription.callback(requestId, slotIndex) - -proc emitSlotFreed*(market: MockMarket, requestId: RequestId, slotIndex: uint64) = - var subscriptions = market.subscriptions.onSlotFreed - for subscription in subscriptions: - subscription.callback(requestId, slotIndex) - -proc emitSlotReservationsFull*( - market: MockMarket, requestId: RequestId, slotIndex: uint64 -) = - var subscriptions = market.subscriptions.onSlotReservationsFull - for subscription in subscriptions: - subscription.callback(requestId, slotIndex) - -proc emitRequestCancelled*(market: MockMarket, requestId: RequestId) = - var subscriptions = market.subscriptions.onRequestCancelled - for subscription in subscriptions: - if subscription.requestId == requestId.some or subscription.requestId.isNone: - subscription.callback(requestId) - -proc emitRequestFulfilled*(market: MockMarket, requestId: RequestId) = - var subscriptions = market.subscriptions.onFulfillment - for subscription in subscriptions: - if subscription.requestId == requestId.some or subscription.requestId.isNone: - subscription.callback(requestId) - -proc emitRequestFailed*(market: MockMarket, requestId: RequestId) = - var subscriptions = market.subscriptions.onRequestFailed - for subscription in subscriptions: - if subscription.requestId == requestId.some or subscription.requestId.isNone: - subscription.callback(requestId) - -proc fillSlot*( - market: MockMarket, - requestId: RequestId, - slotIndex: uint64, - proof: Groth16Proof, - host: Address, - collateral = 0.u256, -) = - if error =? market.errorOnFillSlot: - raise error - - let slot = MockSlot( - requestId: requestId, - slotIndex: slotIndex, - proof: proof, - host: host, - timestamp: market.clock.now, - collateral: collateral, - ) - market.filled.add(slot) - market.slotState[slotId(slot.requestId, slot.slotIndex)] = SlotState.Filled - market.emitSlotFilled(requestId, slotIndex) - -method fillSlot*( - market: MockMarket, - requestId: RequestId, - slotIndex: uint64, - proof: Groth16Proof, - collateral: UInt256, -) {.async: (raises: [CancelledError, MarketError]).} = - market.fillSlot(requestId, slotIndex, proof, market.signer, collateral) - -method freeSlot*( - market: MockMarket, slotId: SlotId -) {.async: (raises: [CancelledError, MarketError]).} = - if error =? market.errorOnFreeSlot: - raise error - - market.freed.add(slotId) - for s in market.filled: - if slotId(s.requestId, s.slotIndex) == slotId: - market.emitSlotFreed(s.requestId, s.slotIndex) - break - market.slotState[slotId] = SlotState.Free - -method withdrawFunds*( - market: MockMarket, requestId: RequestId -) {.async: (raises: [CancelledError, MarketError]).} = - market.withdrawn.add(requestId) - - if state =? market.requestState .? [requestId] and state == RequestState.Cancelled: - market.emitRequestCancelled(requestId) - -proc setProofRequired*(mock: MockMarket, id: SlotId, required: bool) = - if required: - mock.proofsRequired.incl(id) - else: - mock.proofsRequired.excl(id) - -method isProofRequired*(mock: MockMarket, id: SlotId): Future[bool] {.async.} = - return mock.proofsRequired.contains(id) - -proc setProofToBeRequired*(mock: MockMarket, id: SlotId, required: bool) = - if required: - mock.proofsToBeRequired.incl(id) - else: - mock.proofsToBeRequired.excl(id) - -method willProofBeRequired*(mock: MockMarket, id: SlotId): Future[bool] {.async.} = - return mock.proofsToBeRequired.contains(id) - -method getChallenge*(mock: MockMarket, id: SlotId): Future[ProofChallenge] {.async.} = - return mock.proofChallenge - -proc setProofEnd*(mock: MockMarket, id: SlotId, proofEnd: UInt256) = - mock.proofEnds[id] = proofEnd - -method submitProof*( - mock: MockMarket, id: SlotId, proof: Groth16Proof -) {.async: (raises: [CancelledError, MarketError]).} = - mock.submitted.add(proof) - for subscription in mock.subscriptions.onProofSubmitted: - subscription.callback(id) - -method markProofAsMissing*( - market: MockMarket, id: SlotId, period: Period -) {.async: (raises: [CancelledError, MarketError]).} = - market.markedAsMissingProofs.add(id) - -proc setCanMarkProofAsMissing*(mock: MockMarket, id: SlotId, required: bool) = - if required: - mock.canBeMarkedAsMissing.incl(id) - else: - mock.canBeMarkedAsMissing.excl(id) - -method canMarkProofAsMissing*( - market: MockMarket, id: SlotId, period: Period -): Future[bool] {.async: (raises: [CancelledError]).} = - return market.canBeMarkedAsMissing.contains(id) - -method reserveSlot*( - market: MockMarket, requestId: RequestId, slotIndex: uint64 -) {.async: (raises: [CancelledError, MarketError]).} = - if error =? market.errorOnReserveSlot: - raise error - -method canReserveSlot*( - market: MockMarket, requestId: RequestId, slotIndex: uint64 -): Future[bool] {.async.} = - return market.canReserveSlot - -func setCanReserveSlot*(market: MockMarket, canReserveSlot: bool) = - market.canReserveSlot = canReserveSlot - -func setErrorOnReserveSlot*(market: MockMarket, error: ref MarketError) = - market.errorOnReserveSlot = - if error.isNil: - none (ref MarketError) - else: - some error - -func setErrorOnFillSlot*(market: MockMarket, error: ref MarketError) = - market.errorOnFillSlot = - if error.isNil: - none (ref MarketError) - else: - some error - -func setErrorOnFreeSlot*(market: MockMarket, error: ref MarketError) = - market.errorOnFreeSlot = - if error.isNil: - none (ref MarketError) - else: - some error - -func setErrorOnGetHost*(market: MockMarket, error: ref MarketError) = - market.errorOnGetHost = - if error.isNil: - none (ref MarketError) - else: - some error - -method subscribeRequests*( - market: MockMarket, callback: OnRequest -): Future[Subscription] {.async.} = - let subscription = RequestSubscription(market: market, callback: callback) - market.subscriptions.onRequest.add(subscription) - return subscription - -method subscribeFulfillment*( - market: MockMarket, callback: OnFulfillment -): Future[Subscription] {.async.} = - let subscription = FulfillmentSubscription( - market: market, requestId: none RequestId, callback: callback - ) - market.subscriptions.onFulfillment.add(subscription) - return subscription - -method subscribeFulfillment*( - market: MockMarket, requestId: RequestId, callback: OnFulfillment -): Future[Subscription] {.async.} = - let subscription = FulfillmentSubscription( - market: market, requestId: some requestId, callback: callback - ) - market.subscriptions.onFulfillment.add(subscription) - return subscription - -method subscribeSlotFilled*( - market: MockMarket, callback: OnSlotFilled -): Future[Subscription] {.async.} = - let subscription = SlotFilledSubscription(market: market, callback: callback) - market.subscriptions.onSlotFilled.add(subscription) - return subscription - -method subscribeSlotFilled*( - market: MockMarket, requestId: RequestId, slotIndex: uint64, callback: OnSlotFilled -): Future[Subscription] {.async.} = - let subscription = SlotFilledSubscription( - market: market, - requestId: some requestId, - slotIndex: some slotIndex, - callback: callback, - ) - market.subscriptions.onSlotFilled.add(subscription) - return subscription - -method subscribeSlotFreed*( - market: MockMarket, callback: OnSlotFreed -): Future[Subscription] {.async.} = - let subscription = SlotFreedSubscription(market: market, callback: callback) - market.subscriptions.onSlotFreed.add(subscription) - return subscription - -method subscribeSlotReservationsFull*( - market: MockMarket, callback: OnSlotReservationsFull -): Future[Subscription] {.async.} = - let subscription = - SlotReservationsFullSubscription(market: market, callback: callback) - market.subscriptions.onSlotReservationsFull.add(subscription) - return subscription - -method subscribeRequestCancelled*( - market: MockMarket, callback: OnRequestCancelled -): Future[Subscription] {.async.} = - let subscription = RequestCancelledSubscription( - market: market, requestId: none RequestId, callback: callback - ) - market.subscriptions.onRequestCancelled.add(subscription) - return subscription - -method subscribeRequestCancelled*( - market: MockMarket, requestId: RequestId, callback: OnRequestCancelled -): Future[Subscription] {.async.} = - let subscription = RequestCancelledSubscription( - market: market, requestId: some requestId, callback: callback - ) - market.subscriptions.onRequestCancelled.add(subscription) - return subscription - -method subscribeRequestFailed*( - market: MockMarket, callback: OnRequestFailed -): Future[Subscription] {.async.} = - let subscription = RequestFailedSubscription( - market: market, requestId: none RequestId, callback: callback - ) - market.subscriptions.onRequestFailed.add(subscription) - return subscription - -method subscribeRequestFailed*( - market: MockMarket, requestId: RequestId, callback: OnRequestFailed -): Future[Subscription] {.async.} = - let subscription = RequestFailedSubscription( - market: market, requestId: some requestId, callback: callback - ) - market.subscriptions.onRequestFailed.add(subscription) - return subscription - -method subscribeProofSubmission*( - mock: MockMarket, callback: OnProofSubmitted -): Future[Subscription] {.async.} = - let subscription = ProofSubmittedSubscription(market: mock, callback: callback) - mock.subscriptions.onProofSubmitted.add(subscription) - return subscription - -method queryPastStorageRequestedEvents*( - market: MockMarket, fromBlock: BlockTag -): Future[seq[StorageRequested]] {.async.} = - return market.requested.map( - request => - StorageRequested( - requestId: request.id, - ask: request.ask, - expiry: market.requestExpiry[request.id].uint64, - ) - ) - -method queryPastStorageRequestedEvents*( - market: MockMarket, blocksAgo: int -): Future[seq[StorageRequested]] {.async.} = - return market.requested.map( - request => - StorageRequested( - requestId: request.id, - ask: request.ask, - expiry: market.requestExpiry[request.id].uint64, - ) - ) - -method queryPastSlotFilledEvents*( - market: MockMarket, fromBlock: BlockTag -): Future[seq[SlotFilled]] {.async.} = - return market.filled.map( - slot => SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) - ) - -method queryPastSlotFilledEvents*( - market: MockMarket, blocksAgo: int -): Future[seq[SlotFilled]] {.async.} = - return market.filled.map( - slot => SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) - ) - -method queryPastSlotFilledEvents*( - market: MockMarket, fromTime: SecondsSince1970 -): Future[seq[SlotFilled]] {.async.} = - let filtered = market.filled.filter( - proc(slot: MockSlot): bool = - return slot.timestamp >= fromTime - ) - return filtered.map( - slot => SlotFilled(requestId: slot.requestId, slotIndex: slot.slotIndex) - ) - -method unsubscribe*(subscription: RequestSubscription) {.async.} = - subscription.market.subscriptions.onRequest.keepItIf(it != subscription) - -method unsubscribe*(subscription: FulfillmentSubscription) {.async.} = - subscription.market.subscriptions.onFulfillment.keepItIf(it != subscription) - -method unsubscribe*(subscription: SlotFilledSubscription) {.async.} = - subscription.market.subscriptions.onSlotFilled.keepItIf(it != subscription) - -method unsubscribe*(subscription: SlotFreedSubscription) {.async.} = - subscription.market.subscriptions.onSlotFreed.keepItIf(it != subscription) - -method unsubscribe*(subscription: RequestCancelledSubscription) {.async.} = - subscription.market.subscriptions.onRequestCancelled.keepItIf(it != subscription) - -method unsubscribe*(subscription: RequestFailedSubscription) {.async.} = - subscription.market.subscriptions.onRequestFailed.keepItIf(it != subscription) - -method unsubscribe*(subscription: ProofSubmittedSubscription) {.async.} = - subscription.market.subscriptions.onProofSubmitted.keepItIf(it != subscription) - -method unsubscribe*(subscription: SlotReservationsFullSubscription) {.async.} = - subscription.market.subscriptions.onSlotReservationsFull.keepItIf(it != subscription) - -method slotCollateral*( - market: MockMarket, requestId: RequestId, slotIndex: uint64 -): Future[?!UInt256] {.async: (raises: [CancelledError]).} = - let slotid = slotId(requestId, slotIndex) - - try: - let state = await slotState(market, slotid) - - without request =? await market.getRequest(requestId): - return failure newException( - MarketError, "Failure calculating the slotCollateral, cannot get the request" - ) - - return market.slotCollateral(request.ask.collateralPerSlot, state) - except MarketError as error: - error "Error when trying to calculate the slotCollateral", error = error.msg - return failure error - -method slotCollateral*( - market: MockMarket, collateralPerSlot: UInt256, slotState: SlotState -): ?!UInt256 {.raises: [].} = - if slotState == SlotState.Repair: - let repairRewardPercentage = market.config.collateral.repairRewardPercentage.u256 - - return success ( - collateralPerSlot - (collateralPerSlot * repairRewardPercentage).div(100.u256) - ) - - return success collateralPerSlot diff --git a/tests/codex/helpers/mockreservations.nim b/tests/codex/helpers/mockreservations.nim deleted file mode 100644 index a8933e00..00000000 --- a/tests/codex/helpers/mockreservations.nim +++ /dev/null @@ -1,51 +0,0 @@ -import pkg/chronos -import pkg/codex/sales -import pkg/codex/stores -import pkg/questionable/results -import pkg/codex/clock - -type MockReservations* = ref object of Reservations - createReservationThrowBytesOutOfBoundsError: bool - createReservationThrowError: ?(ref CatchableError) - -proc new*(T: type MockReservations, repo: RepoStore): MockReservations = - ## Create a mock clock instance - MockReservations(availabilityLock: newAsyncLock(), repo: repo) - -proc setCreateReservationThrowBytesOutOfBoundsError*( - self: MockReservations, flag: bool -) = - self.createReservationThrowBytesOutOfBoundsError = flag - -proc setCreateReservationThrowError*( - self: MockReservations, error: ?(ref CatchableError) -) = - self.createReservationThrowError = error - -method createReservation*( - self: MockReservations, - availabilityId: AvailabilityId, - slotSize: uint64, - requestId: RequestId, - slotIndex: uint64, - collateralPerByte: UInt256, - validUntil: SecondsSince1970, -): Future[?!Reservation] {.async: (raises: [CancelledError]).} = - if self.createReservationThrowBytesOutOfBoundsError: - let error = newException( - BytesOutOfBoundsError, - "trying to reserve an amount of bytes that is greater than the total size of the Availability", - ) - return failure(error) - elif error =? self.createReservationThrowError: - return failure(error) - - return await procCall createReservation( - Reservations(self), - availabilityId, - slotSize, - requestId, - slotIndex, - collateralPerByte, - validUntil, - ) diff --git a/tests/codex/helpers/mocksalesagent.nim b/tests/codex/helpers/mocksalesagent.nim deleted file mode 100644 index d5de265a..00000000 --- a/tests/codex/helpers/mocksalesagent.nim +++ /dev/null @@ -1,17 +0,0 @@ -import pkg/codex/sales/salesagent - -type MockSalesAgent = ref object of SalesAgent - fulfilledCalled*: bool - failedCalled*: bool - slotFilledCalled*: bool - -method onFulfilled*(agent: SalesAgent, requestId: RequestId) = - fulfilledCalled = true - -method onFailed*(agent: SalesAgent, requestId: RequestId) = - failedCalled = true - -method onSlotFilled*( - agent: SalesAgent, requestId: RequestId, slotIndex: uint64 -) {.base.} = - slotFilledCalled = true diff --git a/tests/codex/helpers/mockslotqueueitem.nim b/tests/codex/helpers/mockslotqueueitem.nim deleted file mode 100644 index 8657850f..00000000 --- a/tests/codex/helpers/mockslotqueueitem.nim +++ /dev/null @@ -1,26 +0,0 @@ -import pkg/codex/contracts/requests -import pkg/codex/sales/slotqueue - -type MockSlotQueueItem* = object - requestId*: RequestId - slotIndex*: uint16 - slotSize*: uint64 - duration*: uint64 - pricePerBytePerSecond*: UInt256 - collateral*: UInt256 - expiry*: uint64 - seen*: bool - -proc toSlotQueueItem*(item: MockSlotQueueItem): SlotQueueItem = - SlotQueueItem.init( - requestId = item.requestId, - slotIndex = item.slotIndex, - ask = StorageAsk( - slotSize: item.slotSize, - duration: item.duration, - pricePerBytePerSecond: item.pricePerBytePerSecond, - ), - expiry = item.expiry, - seen = item.seen, - collateral = item.collateral, - ) diff --git a/tests/codex/helpers/nodeutils.nim b/tests/codex/helpers/nodeutils.nim index 12c38350..55ab984b 100644 --- a/tests/codex/helpers/nodeutils.nim +++ b/tests/codex/helpers/nodeutils.nim @@ -14,7 +14,6 @@ import pkg/codex/systemclock import pkg/codex/nat import pkg/codex/utils/natutils import pkg/codex/utils/safeasynciter -import pkg/codex/slots import pkg/codex/merkletree import pkg/codex/manifest @@ -43,7 +42,6 @@ type NodesComponents* = object switch*: Switch blockDiscovery*: Discovery - wallet*: WalletRef network*: BlockExcNetwork localStore*: BlockStore peerStore*: PeerCtxStore @@ -71,7 +69,6 @@ converter toTuple*( ): tuple[ switch: Switch, blockDiscovery: Discovery, - wallet: WalletRef, network: BlockExcNetwork, localStore: BlockStore, peerStore: PeerCtxStore, @@ -81,7 +78,7 @@ converter toTuple*( networkStore: NetworkStore, ] = ( - nc.switch, nc.blockDiscovery, nc.wallet, nc.network, nc.localStore, nc.peerStore, + nc.switch, nc.blockDiscovery, nc.network, nc.localStore, nc.peerStore, nc.pendingBlocks, nc.discovery, nc.engine, nc.networkStore, ) @@ -164,11 +161,6 @@ proc generateNodes*( MultiAddress.init("/ip4/127.0.0.1/tcp/0").expect("invalid multiaddress"), ) - wallet = - if config.createFullNode: - WalletRef.new(EthPrivateKey.random()) - else: - WalletRef.example network = BlockExcNetwork.new(switch) peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() @@ -209,7 +201,7 @@ proc generateNodes*( ) advertiser = Advertiser.new(localStore, blockDiscovery) engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + localStore, network, discovery, advertiser, peerStore, pendingBlocks ) networkStore = NetworkStore.new(engine, localStore) @@ -221,7 +213,6 @@ proc generateNodes*( switch = switch, networkStore = networkStore, engine = engine, - prover = Prover.none, discovery = blockDiscovery, taskpool = taskpool, ) @@ -245,7 +236,6 @@ proc generateNodes*( let nodeComponent = NodesComponents( switch: switch, blockDiscovery: blockDiscovery, - wallet: wallet, network: network, localStore: localStore, peerStore: peerStore, diff --git a/tests/codex/node/helpers.nim b/tests/codex/node/helpers.nim index a28a1f37..4c318246 100644 --- a/tests/codex/node/helpers.nim +++ b/tests/codex/node/helpers.nim @@ -64,7 +64,6 @@ template setupAndTearDown*() {.dirty.} = file: File chunker: Chunker switch: Switch - wallet: WalletRef network: BlockExcNetwork clock: Clock localStore: RepoStore @@ -88,7 +87,6 @@ template setupAndTearDown*() {.dirty.} = file = open(path /../ "" /../ "fixtures" / "test.jpg") chunker = FileChunker.new(file = file, chunkSize = DefaultBlockSize) switch = newStandardSwitch() - wallet = WalletRef.new(EthPrivateKey.random()) network = BlockExcNetwork.new(switch) clock = SystemClock.new() @@ -110,14 +108,13 @@ template setupAndTearDown*() {.dirty.} = DiscoveryEngine.new(localStore, peerStore, network, blockDiscovery, pendingBlocks) advertiser = Advertiser.new(localStore, blockDiscovery) engine = BlockExcEngine.new( - localStore, wallet, network, discovery, advertiser, peerStore, pendingBlocks + localStore, network, discovery, advertiser, peerStore, pendingBlocks ) store = NetworkStore.new(engine, localStore) node = CodexNodeRef.new( switch = switch, networkStore = store, engine = engine, - prover = Prover.none, discovery = blockDiscovery, taskpool = Taskpool.new(), ) diff --git a/tests/codex/node/testcontracts.nim b/tests/codex/node/testcontracts.nim deleted file mode 100644 index e8d9c743..00000000 --- a/tests/codex/node/testcontracts.nim +++ /dev/null @@ -1,143 +0,0 @@ -import std/os -import std/options -import std/times -import std/importutils - -import pkg/chronos -import pkg/datastore -import pkg/datastore/typedds -import pkg/questionable -import pkg/questionable/results -import pkg/stint -import pkg/taskpools - -import pkg/nitro -import pkg/codexdht/discv5/protocol as discv5 - -import pkg/codex/logutils -import pkg/codex/stores -import pkg/codex/clock -import pkg/codex/contracts -import pkg/codex/systemclock -import pkg/codex/blockexchange -import pkg/codex/chunker -import pkg/codex/slots -import pkg/codex/manifest -import pkg/codex/discovery -import pkg/codex/erasure -import pkg/codex/blocktype as bt -import pkg/codex/stores/repostore/coders -import pkg/codex/utils/asynciter -import pkg/codex/indexingstrategy - -import pkg/codex/node {.all.} - -import ../../asynctest -import ../../examples -import ../helpers -import ../helpers/mockmarket -import ../helpers/mockclock - -import ./helpers - -privateAccess(CodexNodeRef) # enable access to private fields - -asyncchecksuite "Test Node - Host contracts": - setupAndTearDown() - - var - sales: Sales - purchasing: Purchasing - manifest: Manifest - manifestCidStr: string - manifestCid: Cid - market: MockMarket - builder: Poseidon2Builder - verifiable: Manifest - verifiableBlock: bt.Block - protected: Manifest - - setup: - # Setup Host Contracts and dependencies - market = MockMarket.new() - sales = Sales.new(market, clock, localStore) - - node.contracts = ( - none ClientInteractions, - some HostInteractions.new(clock, sales), - none ValidatorInteractions, - ) - - await node.start() - - # Populate manifest in local store - manifest = await storeDataGetManifest(localStore, chunker) - let - manifestBlock = - bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, Taskpool.new) - - manifestCid = manifestBlock.cid - - (await localStore.putBlock(manifestBlock)).tryGet() - - protected = (await erasure.encode(manifest, 3, 2)).tryGet() - builder = Poseidon2Builder.new(localStore, protected).tryGet() - verifiable = (await builder.buildManifest()).tryGet() - verifiableBlock = - bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet() - - (await localStore.putBlock(verifiableBlock)).tryGet() - - test "onExpiryUpdate callback is set": - check sales.onExpiryUpdate.isSome - - test "onExpiryUpdate callback": - let - # The blocks have set default TTL, so in order to update it we have to have larger TTL - expectedExpiry: SecondsSince1970 = clock.now + DefaultBlockTtl.seconds + 11123 - expiryUpdateCallback = !sales.onExpiryUpdate - - (await expiryUpdateCallback(manifestCid, expectedExpiry)).tryGet() - - for index in 0 ..< manifest.blocksCount: - let - blk = (await localStore.getBlock(manifest.treeCid, index)).tryGet - key = (createBlockExpirationMetadataKey(blk.cid)).tryGet - bytes = (await localStoreMetaDs.get(key)).tryGet - blkMd = BlockMetadata.decode(bytes).tryGet - - check blkMd.expiry == expectedExpiry - - test "onStore callback is set": - check sales.onStore.isSome - - test "onStore callback": - let onStore = !sales.onStore - var request = StorageRequest.example - request.content.cid = verifiableBlock.cid - let expiry = (getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix - var fetchedBytes: uint = 0 - - let onBlocks = proc( - blocks: seq[bt.Block] - ): Future[?!void] {.async: (raises: [CancelledError]).} = - for blk in blocks: - fetchedBytes += blk.data.len.uint - return success() - - (await onStore(request, expiry, 1.uint64, onBlocks, isRepairing = false)).tryGet() - check fetchedBytes == 12 * DefaultBlockSize.uint - - let indexer = verifiable.verifiableStrategy.init( - 0, verifiable.blocksCount - 1, verifiable.numSlots - ) - - for index in indexer.getIndices(1): - let - blk = (await localStore.getBlock(verifiable.treeCid, index)).tryGet - key = (createBlockExpirationMetadataKey(blk.cid)).tryGet - bytes = (await localStoreMetaDs.get(key)).tryGet - blkMd = BlockMetadata.decode(bytes).tryGet - - check blkMd.expiry == expiry diff --git a/tests/codex/node/testnode.nim b/tests/codex/node/testnode.nim index 09ed7ca7..6be87ff4 100644 --- a/tests/codex/node/testnode.nim +++ b/tests/codex/node/testnode.nim @@ -14,20 +14,16 @@ import pkg/poseidon2 import pkg/poseidon2/io import pkg/taskpools -import pkg/nitro import pkg/codexdht/discv5/protocol as discv5 import pkg/codex/logutils import pkg/codex/stores import pkg/codex/clock -import pkg/codex/contracts import pkg/codex/systemclock import pkg/codex/blockexchange import pkg/codex/chunker -import pkg/codex/slots import pkg/codex/manifest import pkg/codex/discovery -import pkg/codex/erasure import pkg/codex/merkletree import pkg/codex/blocktype as bt import pkg/codex/rng @@ -37,7 +33,6 @@ import pkg/codex/node {.all.} import ../../asynctest import ../examples import ../helpers -import ../helpers/mockmarket import ../helpers/mockclock import ../slots/helpers @@ -172,39 +167,6 @@ asyncchecksuite "Test Node - Basic": await stream.readExactly(addr data[0], data.len) check string.fromBytes(data) == testString - test "Setup purchase request": - let - erasure = - Erasure.new(store, leoEncoderProvider, leoDecoderProvider, Taskpool.new()) - manifest = await storeDataGetManifest(localStore, chunker) - manifestBlock = - bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - protected = (await erasure.encode(manifest, 3, 2)).tryGet() - builder = Poseidon2Builder.new(localStore, protected).tryGet() - verifiable = (await builder.buildManifest()).tryGet() - verifiableBlock = - bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet() - - (await localStore.putBlock(manifestBlock)).tryGet() - - let request = ( - await node.setupRequest( - cid = manifestBlock.cid, - nodes = 5, - tolerance = 2, - duration = 100.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - expiry = 200.uint64, - collateralPerByte = 1.u256, - ) - ).tryGet - - check: - (await verifiableBlock.cid in localStore) == true - request.content.cid == verifiableBlock.cid - request.content.merkleRoot == builder.verifyRoot.get.toBytes - test "Should delete a single block": let randomBlock = bt.Block.new("Random block".toBytes).tryGet() (await localStore.putBlock(randomBlock)).tryGet() diff --git a/tests/codex/node/testslotrepair.nim b/tests/codex/node/testslotrepair.nim deleted file mode 100644 index 2c778f44..00000000 --- a/tests/codex/node/testslotrepair.nim +++ /dev/null @@ -1,221 +0,0 @@ -import std/options -import std/importutils -import std/times - -import pkg/chronos -import pkg/questionable -import pkg/questionable/results -import pkg/stint - -import pkg/codex/logutils -import pkg/codex/stores -import pkg/codex/contracts -import pkg/codex/slots -import pkg/codex/manifest -import pkg/codex/erasure -import pkg/codex/blocktype as bt -import pkg/chronos/transports/stream - -import pkg/codex/node {.all.} - -import ../../asynctest -import ../../examples -import ../helpers - -import ./helpers - -privateAccess(CodexNodeRef) # enable access to private fields - -logScope: - topics = "testSlotRepair" - -proc fetchStreamData(stream: LPStream, datasetSize: int): Future[seq[byte]] {.async.} = - var buf = newSeq[byte](datasetSize) - await stream.readExactly(addr buf[0], datasetSize) - buf - -proc flatten[T](s: seq[seq[T]]): seq[T] = - var t = newSeq[T](0) - for ss in s: - t &= ss - return t - -asyncchecksuite "Test Node - Slot Repair": - let - numNodes = 12 - config = NodeConfig( - useRepoStore: true, - findFreePorts: true, - createFullNode: true, - enableBootstrap: true, - enableDiscovery: true, - ) - var - manifest: Manifest - builder: Poseidon2Builder - verifiable: Manifest - verifiableBlock: bt.Block - protected: Manifest - cluster: NodesCluster - - nodes: seq[CodexNodeRef] - localStores: seq[BlockStore] - - setup: - cluster = generateNodes(numNodes, config = config) - nodes = cluster.nodes - localStores = cluster.localStores - - teardown: - await cluster.cleanup() - localStores = @[] - nodes = @[] - - test "repair slots (2,1)": - let - expiry = (getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix - numBlocks = 5 - datasetSize = numBlocks * DefaultBlockSize.int - ecK = 2 - ecM = 1 - localStore = localStores[0] - store = nodes[0].blockStore - blocks = - await makeRandomBlocks(datasetSize = datasetSize, blockSize = DefaultBlockSize) - data = ( - block: - collect(newSeq): - for blk in blocks: - blk.data - ).flatten() - check blocks.len == numBlocks - - # Populate manifest in local store - manifest = await storeDataGetManifest(localStore, blocks) - let - manifestBlock = - bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - erasure = - Erasure.new(store, leoEncoderProvider, leoDecoderProvider, cluster.taskpool) - - (await localStore.putBlock(manifestBlock)).tryGet() - - protected = (await erasure.encode(manifest, ecK, ecM)).tryGet() - builder = Poseidon2Builder.new(localStore, protected).tryGet() - verifiable = (await builder.buildManifest()).tryGet() - verifiableBlock = - bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet() - - # Populate protected manifest in local store - (await localStore.putBlock(verifiableBlock)).tryGet() - - var request = StorageRequest.example - request.content.cid = verifiableBlock.cid - - for i in 0 ..< protected.numSlots.uint64: - (await nodes[i + 1].onStore(request, expiry, i, nil, isRepairing = false)).tryGet() - - await nodes[0].switch.stop() # acts as client - await nodes[1].switch.stop() # slot 0 missing now - - # repair missing slot - (await nodes[4].onStore(request, expiry, 0.uint64, nil, isRepairing = true)).tryGet() - - await nodes[2].switch.stop() # slot 1 missing now - - (await nodes[5].onStore(request, expiry, 1.uint64, nil, isRepairing = true)).tryGet() - - await nodes[3].switch.stop() # slot 2 missing now - - (await nodes[6].onStore(request, expiry, 2.uint64, nil, isRepairing = true)).tryGet() - - await nodes[4].switch.stop() # slot 0 missing now - - # repair missing slot from repaired slots - (await nodes[7].onStore(request, expiry, 0.uint64, nil, isRepairing = true)).tryGet() - - await nodes[5].switch.stop() # slot 1 missing now - - # repair missing slot from repaired slots - (await nodes[8].onStore(request, expiry, 1.uint64, nil, isRepairing = true)).tryGet() - - await nodes[6].switch.stop() # slot 2 missing now - - # repair missing slot from repaired slots - (await nodes[9].onStore(request, expiry, 2.uint64, nil, isRepairing = true)).tryGet() - - let - stream = (await nodes[10].retrieve(verifiableBlock.cid, local = false)).tryGet() - expectedData = await fetchStreamData(stream, datasetSize) - check expectedData.len == data.len - check expectedData == data - - test "repair slots (3,2)": - let - expiry = (getTime() + DefaultBlockTtl.toTimesDuration + 1.hours).toUnix - numBlocks = 40 - datasetSize = numBlocks * DefaultBlockSize.int - ecK = 3 - ecM = 2 - localStore = localStores[0] - store = nodes[0].blockStore - blocks = - await makeRandomBlocks(datasetSize = datasetSize, blockSize = DefaultBlockSize) - data = ( - block: - collect(newSeq): - for blk in blocks: - blk.data - ).flatten() - check blocks.len == numBlocks - - # Populate manifest in local store - manifest = await storeDataGetManifest(localStore, blocks) - let - manifestBlock = - bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - erasure = - Erasure.new(store, leoEncoderProvider, leoDecoderProvider, cluster.taskpool) - - (await localStore.putBlock(manifestBlock)).tryGet() - - protected = (await erasure.encode(manifest, ecK, ecM)).tryGet() - builder = Poseidon2Builder.new(localStore, protected).tryGet() - verifiable = (await builder.buildManifest()).tryGet() - verifiableBlock = - bt.Block.new(verifiable.encode().tryGet(), codec = ManifestCodec).tryGet() - - # Populate protected manifest in local store - (await localStore.putBlock(verifiableBlock)).tryGet() - - var request = StorageRequest.example - request.content.cid = verifiableBlock.cid - - for i in 0 ..< protected.numSlots.uint64: - (await nodes[i + 1].onStore(request, expiry, i, nil, isRepairing = false)).tryGet() - - await nodes[0].switch.stop() # acts as client - await nodes[1].switch.stop() # slot 0 missing now - await nodes[3].switch.stop() # slot 2 missing now - - # repair missing slots - (await nodes[6].onStore(request, expiry, 0.uint64, nil, isRepairing = true)).tryGet() - (await nodes[7].onStore(request, expiry, 2.uint64, nil, isRepairing = true)).tryGet() - - await nodes[2].switch.stop() # slot 1 missing now - await nodes[4].switch.stop() # slot 3 missing now - - # repair missing slots from repaired slots - (await nodes[8].onStore(request, expiry, 1.uint64, nil, isRepairing = true)).tryGet() - (await nodes[9].onStore(request, expiry, 3.uint64, nil, isRepairing = true)).tryGet() - - await nodes[5].switch.stop() # slot 4 missing now - - # repair missing slot from repaired slots - (await nodes[10].onStore(request, expiry, 4.uint64, nil, isRepairing = true)).tryGet() - - let - stream = (await nodes[11].retrieve(verifiableBlock.cid, local = false)).tryGet() - expectedData = await fetchStreamData(stream, datasetSize) - check expectedData.len == data.len - check expectedData == data diff --git a/tests/codex/sales/helpers/periods.nim b/tests/codex/sales/helpers/periods.nim deleted file mode 100644 index 99716cec..00000000 --- a/tests/codex/sales/helpers/periods.nim +++ /dev/null @@ -1,8 +0,0 @@ -import pkg/codex/market -import ../../helpers/mockclock - -proc advanceToNextPeriod*(clock: MockClock, market: Market) {.async.} = - let periodicity = await market.periodicity() - let period = periodicity.periodOf(clock.now().Timestamp) - let periodEnd = periodicity.periodEnd(period) - clock.set(periodEnd.toSecondsSince1970 + 1) diff --git a/tests/codex/sales/states/testcancelled.nim b/tests/codex/sales/states/testcancelled.nim deleted file mode 100644 index 972051d4..00000000 --- a/tests/codex/sales/states/testcancelled.nim +++ /dev/null @@ -1,113 +0,0 @@ -import pkg/questionable -import pkg/chronos -import pkg/codex/contracts/requests -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/errored -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/market -from pkg/codex/utils/asyncstatemachine import State - -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock - -asyncchecksuite "sales state 'cancelled'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - let clock = MockClock.new() - - let currentCollateral = UInt256.example - - var market: MockMarket - var state: SaleCancelled - var agent: SalesAgent - var reprocessSlotWas: ?bool - var returnedCollateralValue: ?UInt256 - - setup: - market = MockMarket.new() - let onCleanUp = proc( - reprocessSlot = false, returnedCollateral = UInt256.none - ) {.async: (raises: []).} = - reprocessSlotWas = some reprocessSlot - returnedCollateralValue = returnedCollateral - - let context = SalesContext(market: market, clock: clock) - agent = newSalesAgent(context, request.id, slotIndex, request.some) - agent.onCleanUp = onCleanUp - state = SaleCancelled.new() - reprocessSlotWas = bool.none - returnedCollateralValue = UInt256.none - teardown: - reprocessSlotWas = bool.none - returnedCollateralValue = UInt256.none - - test "calls onCleanUp with reprocessSlot = true, and returnedCollateral = currentCollateral": - market.fillSlot( - requestId = request.id, - slotIndex = slotIndex, - proof = Groth16Proof.default, - host = await market.getSigner(), - collateral = currentCollateral, - ) - discard await state.run(agent) - check eventually reprocessSlotWas == some false - check eventually returnedCollateralValue == some currentCollateral - - test "completes the cancelled state when free slot error is raised and the collateral is returned when a host is hosting a slot": - market.fillSlot( - requestId = request.id, - slotIndex = slotIndex, - proof = Groth16Proof.default, - host = await market.getSigner(), - collateral = currentCollateral, - ) - - let error = - newException(SlotStateMismatchError, "Failed to free slot, slot is already free") - market.setErrorOnFreeSlot(error) - - let next = await state.run(agent) - check next == none State - check eventually reprocessSlotWas == some false - check eventually returnedCollateralValue == some currentCollateral - - test "completes the cancelled state when free slot error is raised and the collateral is not returned when a host is not hosting a slot": - discard market.reserveSlot(requestId = request.id, slotIndex = slotIndex) - market.fillSlot( - requestId = request.id, - slotIndex = slotIndex, - proof = Groth16Proof.default, - host = Address.example, - collateral = currentCollateral, - ) - - let error = - newException(SlotStateMismatchError, "Failed to free slot, slot is already free") - market.setErrorOnFreeSlot(error) - - let next = await state.run(agent) - check next == none State - check eventually reprocessSlotWas == some false - check eventually returnedCollateralValue == UInt256.none - - test "calls onCleanUp and returns the collateral when an error is raised": - market.fillSlot( - requestId = request.id, - slotIndex = slotIndex, - proof = Groth16Proof.default, - host = Address.example, - collateral = currentCollateral, - ) - - let error = newException(MarketError, "") - market.setErrorOnGetHost(error) - - let next = !(await state.run(agent)) - - check next of SaleErrored - let errored = SaleErrored(next) - check errored.error == error diff --git a/tests/codex/sales/states/testdownloading.nim b/tests/codex/sales/states/testdownloading.nim deleted file mode 100644 index 71376fc8..00000000 --- a/tests/codex/sales/states/testdownloading.nim +++ /dev/null @@ -1,29 +0,0 @@ -import pkg/unittest2 -import pkg/questionable -import pkg/codex/contracts/requests -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/downloading -import pkg/codex/sales/states/failed -import pkg/codex/sales/states/filled -import ../../examples -import ../../helpers - -suite "sales state 'downloading'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - var state: SaleDownloading - - setup: - state = SaleDownloading.new() - - test "switches to cancelled state when request expires": - let next = state.onCancelled(request) - check !next of SaleCancelled - - test "switches to failed state when request fails": - let next = state.onFailed(request) - check !next of SaleFailed - - test "switches to filled state when slot is filled": - let next = state.onSlotFilled(request.id, slotIndex) - check !next of SaleFilled diff --git a/tests/codex/sales/states/testerrored.nim b/tests/codex/sales/states/testerrored.nim deleted file mode 100644 index b0352edb..00000000 --- a/tests/codex/sales/states/testerrored.nim +++ /dev/null @@ -1,39 +0,0 @@ -import pkg/questionable -import pkg/chronos -import pkg/codex/contracts/requests -import pkg/codex/sales/states/errored -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/market - -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock - -asyncchecksuite "sales state 'errored'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - let market = MockMarket.new() - let clock = MockClock.new() - - var state: SaleErrored - var agent: SalesAgent - var reprocessSlotWas = false - - setup: - let onCleanUp = proc( - reprocessSlot = false, returnedCollateral = UInt256.none - ) {.async: (raises: []).} = - reprocessSlotWas = reprocessSlot - - let context = SalesContext(market: market, clock: clock) - agent = newSalesAgent(context, request.id, slotIndex, request.some) - agent.onCleanUp = onCleanUp - state = SaleErrored(error: newException(ValueError, "oh no!")) - - test "calls onCleanUp with reprocessSlot = true": - state = SaleErrored(error: newException(ValueError, "oh no!"), reprocessSlot: true) - discard await state.run(agent) - check eventually reprocessSlotWas == true diff --git a/tests/codex/sales/states/testfilled.nim b/tests/codex/sales/states/testfilled.nim deleted file mode 100644 index caf22ee6..00000000 --- a/tests/codex/sales/states/testfilled.nim +++ /dev/null @@ -1,68 +0,0 @@ -import pkg/questionable/results - -import pkg/codex/clock -import pkg/codex/contracts/requests -import pkg/codex/sales -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/sales/states/filled -import pkg/codex/sales/states/errored -import pkg/codex/sales/states/proving - -import ../../../asynctest -import ../../helpers/mockmarket -import ../../examples -import ../../helpers - -suite "sales state 'filled'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - - var market: MockMarket - var slot: MockSlot - var agent: SalesAgent - var state: SaleFilled - var onExpiryUpdatePassedExpiry: SecondsSince1970 - - setup: - market = MockMarket.new() - slot = MockSlot( - requestId: request.id, - host: Address.example, - slotIndex: slotIndex, - proof: Groth16Proof.default, - ) - - market.requestEnds[request.id] = 321 - onExpiryUpdatePassedExpiry = -1 - let onExpiryUpdate = proc( - rootCid: Cid, expiry: SecondsSince1970 - ): Future[?!void] {.async: (raises: [CancelledError]).} = - onExpiryUpdatePassedExpiry = expiry - return success() - let context = SalesContext(market: market, onExpiryUpdate: some onExpiryUpdate) - - agent = newSalesAgent(context, request.id, slotIndex, some request) - state = SaleFilled.new() - - test "switches to proving state when slot is filled by me": - slot.host = await market.getSigner() - market.filled = @[slot] - let next = await state.run(agent) - check !next of SaleProving - - test "calls onExpiryUpdate with request end": - slot.host = await market.getSigner() - market.filled = @[slot] - - let expectedExpiry = 123 - market.requestEnds[request.id] = expectedExpiry - let next = await state.run(agent) - check !next of SaleProving - check onExpiryUpdatePassedExpiry == expectedExpiry - - test "switches to error state when slot is filled by another host": - slot.host = Address.example - market.filled = @[slot] - let next = await state.run(agent) - check !next of SaleErrored diff --git a/tests/codex/sales/states/testfilling.nim b/tests/codex/sales/states/testfilling.nim deleted file mode 100644 index 54536a4c..00000000 --- a/tests/codex/sales/states/testfilling.nim +++ /dev/null @@ -1,61 +0,0 @@ -import pkg/questionable -import pkg/codex/contracts/requests -import pkg/codex/sales/states/filling -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/failed -import pkg/codex/sales/states/ignored -import pkg/codex/sales/states/errored -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock - -suite "sales state 'filling'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - var state: SaleFilling - var market: MockMarket - var clock: MockClock - var agent: SalesAgent - - setup: - clock = MockClock.new() - market = MockMarket.new() - let context = SalesContext(market: market, clock: clock) - agent = newSalesAgent(context, request.id, slotIndex, request.some) - state = SaleFilling.new() - - test "switches to cancelled state when request expires": - let next = state.onCancelled(request) - check !next of SaleCancelled - - test "switches to failed state when request fails": - let next = state.onFailed(request) - check !next of SaleFailed - - test "run switches to ignored when slot is not free": - let error = newException( - SlotStateMismatchError, "Failed to fill slot because the slot is not free" - ) - market.setErrorOnFillSlot(error) - market.requested.add(request) - market.slotState[request.slotId(slotIndex)] = SlotState.Filled - - let next = !(await state.run(agent)) - check next of SaleIgnored - check SaleIgnored(next).reprocessSlot == false - - test "run switches to errored with other error ": - let error = newException(MarketError, "some error") - market.setErrorOnFillSlot(error) - market.requested.add(request) - market.slotState[request.slotId(slotIndex)] = SlotState.Filled - - let next = !(await state.run(agent)) - check next of SaleErrored - - let errored = SaleErrored(next) - check errored.error == error diff --git a/tests/codex/sales/states/testfinished.nim b/tests/codex/sales/states/testfinished.nim deleted file mode 100644 index b5502351..00000000 --- a/tests/codex/sales/states/testfinished.nim +++ /dev/null @@ -1,57 +0,0 @@ -import pkg/questionable -import pkg/codex/contracts/requests -import pkg/codex/sales/states/finished -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/failed -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/market - -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock - -asyncchecksuite "sales state 'finished'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - let clock = MockClock.new() - - let currentCollateral = UInt256.example - - var market: MockMarket - var state: SaleFinished - var agent: SalesAgent - var reprocessSlotWas = bool.none - var returnedCollateralValue = UInt256.none - var saleCleared = bool.none - - setup: - market = MockMarket.new() - let onCleanUp = proc( - reprocessSlot = false, returnedCollateral = UInt256.none - ) {.async: (raises: []).} = - reprocessSlotWas = some reprocessSlot - returnedCollateralValue = returnedCollateral - - let context = SalesContext(market: market, clock: clock) - agent = newSalesAgent(context, request.id, slotIndex, request.some) - agent.onCleanUp = onCleanUp - agent.context.onClear = some proc(request: StorageRequest, idx: uint64) = - saleCleared = some true - state = SaleFinished(returnedCollateral: some currentCollateral) - - test "switches to cancelled state when request expires": - let next = state.onCancelled(request) - check !next of SaleCancelled - - test "switches to failed state when request fails": - let next = state.onFailed(request) - check !next of SaleFailed - - test "calls onCleanUp with reprocessSlot = true, and returnedCollateral = currentCollateral": - discard await state.run(agent) - check eventually reprocessSlotWas == some false - check eventually returnedCollateralValue == some currentCollateral - check eventually saleCleared == some true diff --git a/tests/codex/sales/states/testignored.nim b/tests/codex/sales/states/testignored.nim deleted file mode 100644 index 8b676387..00000000 --- a/tests/codex/sales/states/testignored.nim +++ /dev/null @@ -1,50 +0,0 @@ -import pkg/questionable -import pkg/chronos -import pkg/codex/contracts/requests -import pkg/codex/sales/states/ignored -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/market - -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock - -asyncchecksuite "sales state 'ignored'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - let market = MockMarket.new() - let clock = MockClock.new() - let currentCollateral = UInt256.example - - var state: SaleIgnored - var agent: SalesAgent - var reprocessSlotWas = false - var returnedCollateralValue: ?UInt256 - - setup: - let onCleanUp = proc( - reprocessSlot = false, returnedCollateral = UInt256.none - ) {.async: (raises: []).} = - reprocessSlotWas = reprocessSlot - returnedCollateralValue = returnedCollateral - - let context = SalesContext(market: market, clock: clock) - agent = newSalesAgent(context, request.id, slotIndex, request.some) - agent.onCleanUp = onCleanUp - state = SaleIgnored.new() - returnedCollateralValue = UInt256.none - reprocessSlotWas = false - - test "calls onCleanUp with values assigned to SaleIgnored": - state = SaleIgnored(reprocessSlot: true) - discard await state.run(agent) - check eventually reprocessSlotWas == true - check eventually returnedCollateralValue.isNone - - test "returns collateral when returnsCollateral is true": - state = SaleIgnored(reprocessSlot: false, returnsCollateral: true) - discard await state.run(agent) - check eventually returnedCollateralValue.isSome diff --git a/tests/codex/sales/states/testinitialproving.nim b/tests/codex/sales/states/testinitialproving.nim deleted file mode 100644 index 0e0058d5..00000000 --- a/tests/codex/sales/states/testinitialproving.nim +++ /dev/null @@ -1,102 +0,0 @@ -import pkg/questionable -import pkg/chronos -import pkg/codex/contracts/requests -import pkg/codex/sales/states/initialproving -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/failed -import pkg/codex/sales/states/filling -import pkg/codex/sales/states/errored -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/market - -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock -import ../helpers/periods - -asyncchecksuite "sales state 'initialproving'": - let proof = Groth16Proof.example - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - let market = MockMarket.new() - let clock = MockClock.new() - - var state: SaleInitialProving - var agent: SalesAgent - var receivedChallenge: ProofChallenge - - setup: - let onProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - receivedChallenge = challenge - return success(proof) - let context = SalesContext(onProve: onProve.some, market: market, clock: clock) - agent = newSalesAgent(context, request.id, slotIndex, request.some) - state = SaleInitialProving.new() - - proc allowProofToStart() {.async.} = - # it won't start proving until the next period - await clock.advanceToNextPeriod(market) - - test "switches to cancelled state when request expires": - let next = state.onCancelled(request) - check !next of SaleCancelled - - test "switches to failed state when request fails": - let next = state.onFailed(request) - check !next of SaleFailed - - test "waits for the beginning of the period to get the challenge": - let future = state.run(agent) - check eventually clock.isWaiting - check not future.finished - await allowProofToStart() - discard await future - - test "waits another period when the proof pointer is about to wrap around": - market.proofPointer = 250 - let future = state.run(agent) - await allowProofToStart() - check eventually clock.isWaiting - check not future.finished - market.proofPointer = 100 - await allowProofToStart() - discard await future - - test "onProve callback provides proof challenge": - market.proofChallenge = ProofChallenge.example - - let future = state.run(agent) - await allowProofToStart() - - discard await future - - check receivedChallenge == market.proofChallenge - - test "switches to filling state when initial proving is complete": - let future = state.run(agent) - await allowProofToStart() - let next = await future - - check !next of SaleFilling - check SaleFilling(!next).proof == proof - - test "switches to errored state when onProve callback fails": - let onProveFailed: OnProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - return failure("oh no!") - - let proofFailedContext = - SalesContext(onProve: onProveFailed.some, market: market, clock: clock) - agent = newSalesAgent(proofFailedContext, request.id, slotIndex, request.some) - - let future = state.run(agent) - await allowProofToStart() - let next = await future - - check !next of SaleErrored diff --git a/tests/codex/sales/states/testpayout.nim b/tests/codex/sales/states/testpayout.nim deleted file mode 100644 index 403c663f..00000000 --- a/tests/codex/sales/states/testpayout.nim +++ /dev/null @@ -1,44 +0,0 @@ -import pkg/questionable -import pkg/chronos -import pkg/codex/contracts/requests -import pkg/codex/sales/states/payout -import pkg/codex/sales/states/finished -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/market - -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock - -asyncchecksuite "sales state 'payout'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - let clock = MockClock.new() - - let currentCollateral = UInt256.example - - var market: MockMarket - var state: SalePayout - var agent: SalesAgent - - setup: - market = MockMarket.new() - - let context = SalesContext(market: market, clock: clock) - agent = newSalesAgent(context, request.id, slotIndex, request.some) - state = SalePayout.new() - - test "switches to 'finished' state and provides returnedCollateral": - market.fillSlot( - requestId = request.id, - slotIndex = slotIndex, - proof = Groth16Proof.default, - host = Address.example, - collateral = currentCollateral, - ) - let next = await state.run(agent) - check !next of SaleFinished - check SaleFinished(!next).returnedCollateral == some currentCollateral diff --git a/tests/codex/sales/states/testpreparing.nim b/tests/codex/sales/states/testpreparing.nim deleted file mode 100644 index 74754411..00000000 --- a/tests/codex/sales/states/testpreparing.nim +++ /dev/null @@ -1,125 +0,0 @@ -import pkg/chronos -import pkg/questionable -import pkg/datastore -import pkg/codex/contracts/requests -import pkg/codex/sales/states/preparing -import pkg/codex/sales/states/slotreserving -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/failed -import pkg/codex/sales/states/filled -import pkg/codex/sales/states/ignored -import pkg/codex/sales/states/errored -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/sales/reservations -import pkg/codex/stores/repostore -import times -import ../../../asynctest -import ../../helpers -import ../../examples -import ../../helpers/mockmarket -import ../../helpers/mockreservations -import ../../helpers/mockclock - -asyncchecksuite "sales state 'preparing'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - let market = MockMarket.new() - let clock = MockClock.new() - var agent: SalesAgent - var state: SalePreparing - var repo: RepoStore - var availability: Availability - var context: SalesContext - var reservations: MockReservations - - setup: - availability = Availability.init( - totalSize = request.ask.slotSize + 100.uint64, - freeSize = request.ask.slotSize + 100.uint64, - duration = request.ask.duration + 60.uint64, - minPricePerBytePerSecond = request.ask.pricePerBytePerSecond, - totalCollateral = request.ask.collateralPerSlot * request.ask.slots.u256, - enabled = true, - until = 0.SecondsSince1970, - ) - let repoDs = SQLiteDatastore.new(Memory).tryGet() - let metaDs = SQLiteDatastore.new(Memory).tryGet() - repo = RepoStore.new(repoDs, metaDs) - await repo.start() - - state = SalePreparing.new() - context = SalesContext(market: market, clock: clock) - - reservations = MockReservations.new(repo) - context.reservations = reservations - agent = newSalesAgent(context, request.id, slotIndex, request.some) - - market.requestEnds[request.id] = clock.now() + cast[int64](request.ask.duration) - - teardown: - await repo.stop() - - test "switches to cancelled state when request expires": - let next = state.onCancelled(request) - check !next of SaleCancelled - - test "switches to failed state when request fails": - let next = state.onFailed(request) - check !next of SaleFailed - - test "switches to filled state when slot is filled": - let next = state.onSlotFilled(request.id, slotIndex) - check !next of SaleFilled - - test "run switches to errored when the request cannot be retrieved": - agent = newSalesAgent(context, request.id, slotIndex, StorageRequest.none) - let next = !(await state.run(agent)) - check next of SaleErrored - check SaleErrored(next).error.msg == "request could not be retrieved" - - proc createAvailability(enabled = true) {.async.} = - let a = await reservations.createAvailability( - availability.totalSize, - availability.duration, - availability.minPricePerBytePerSecond, - availability.totalCollateral, - enabled, - until = 0.SecondsSince1970, - ) - availability = a.get - - test "run switches to ignored when no availability": - let next = !(await state.run(agent)) - check next of SaleIgnored - let ignored = SaleIgnored(next) - check ignored.reprocessSlot - - test "run switches to ignored when availability is not enabled": - await createAvailability(enabled = false) - let next = !(await state.run(agent)) - check next of SaleIgnored - - test "run switches to slot reserving state after reservation created": - await createAvailability() - let next = await state.run(agent) - check !next of SaleSlotReserving - - test "run switches to ignored when reserve fails with BytesOutOfBounds": - await createAvailability() - reservations.setCreateReservationThrowBytesOutOfBoundsError(true) - - let next = !(await state.run(agent)) - check next of SaleIgnored - let ignored = SaleIgnored(next) - check ignored.reprocessSlot - - test "run switches to errored when reserve fails with other error": - await createAvailability() - let error = newException(CatchableError, "some error") - reservations.setCreateReservationThrowError(some error) - - let next = !(await state.run(agent)) - check next of SaleErrored - let errored = SaleErrored(next) - check errored.error == error diff --git a/tests/codex/sales/states/testproving.nim b/tests/codex/sales/states/testproving.nim deleted file mode 100644 index ce7b8ba5..00000000 --- a/tests/codex/sales/states/testproving.nim +++ /dev/null @@ -1,105 +0,0 @@ -import pkg/chronos -import pkg/questionable -import pkg/codex/contracts/requests -import pkg/codex/sales/states/proving -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/failed -import pkg/codex/sales/states/payout -import pkg/codex/sales/states/errored -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext - -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock - -asyncchecksuite "sales state 'proving'": - let slot = Slot.example - let request = slot.request - let proof = Groth16Proof.example - - var market: MockMarket - var clock: MockClock - var agent: SalesAgent - var state: SaleProving - var receivedChallenge: ProofChallenge - - setup: - clock = MockClock.new() - market = MockMarket.new() - let onProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - receivedChallenge = challenge - return success(proof) - let context = SalesContext(market: market, clock: clock, onProve: onProve.some) - agent = newSalesAgent(context, request.id, slot.slotIndex, request.some) - state = SaleProving.new() - - proc advanceToNextPeriod(market: Market) {.async.} = - let periodicity = await market.periodicity() - let current = periodicity.periodOf(clock.now().Timestamp) - let periodEnd = periodicity.periodEnd(current) - clock.set(periodEnd.toSecondsSince1970 + 1) - - test "switches to cancelled state when request expires": - let next = state.onCancelled(request) - check !next of SaleCancelled - - test "switches to failed state when request fails": - let next = state.onFailed(request) - check !next of SaleFailed - - test "submits proofs": - var receivedIds: seq[SlotId] - - proc onProofSubmission(id: SlotId) = - receivedIds.add(id) - - let subscription = await market.subscribeProofSubmission(onProofSubmission) - market.slotState[slot.id] = SlotState.Filled - - let future = state.run(agent) - - market.setProofRequired(slot.id, true) - await market.advanceToNextPeriod() - - check eventually receivedIds.contains(slot.id) - - await future.cancelAndWait() - await subscription.unsubscribe() - - test "switches to payout state when request is finished": - market.slotState[slot.id] = SlotState.Filled - - let future = state.run(agent) - - market.slotState[slot.id] = SlotState.Finished - await market.advanceToNextPeriod() - - check eventually future.finished - check !(future.read()) of SalePayout - - test "switches to error state when slot is no longer filled": - market.slotState[slot.id] = SlotState.Filled - - let future = state.run(agent) - - market.slotState[slot.id] = SlotState.Free - await market.advanceToNextPeriod() - - check eventually future.finished - check !(future.read()) of SaleErrored - - test "onProve callback provides proof challenge": - market.proofChallenge = ProofChallenge.example - market.slotState[slot.id] = SlotState.Filled - market.setProofRequired(slot.id, true) - - let future = state.run(agent) - - check eventually receivedChallenge == market.proofChallenge - - await future.cancelAndWait() diff --git a/tests/codex/sales/states/testsimulatedproving.nim b/tests/codex/sales/states/testsimulatedproving.nim deleted file mode 100644 index 304933fe..00000000 --- a/tests/codex/sales/states/testsimulatedproving.nim +++ /dev/null @@ -1,96 +0,0 @@ -import pkg/chronos -import pkg/questionable -import pkg/codex/contracts/requests -import pkg/codex/sales/states/provingsimulated -import pkg/codex/sales/states/proving -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/failed -import pkg/codex/sales/states/payout -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext - -import ../../../asynctest -import ../../examples -import ../../helpers -import ../../helpers/mockmarket -import ../../helpers/mockclock - -asyncchecksuite "sales state 'simulated-proving'": - let slot = Slot.example - let request = slot.request - let proof = Groth16Proof.example - let failEveryNProofs = 3 - let totalProofs = 6 - - var market: MockMarket - var clock: MockClock - var agent: SalesAgent - var state: SaleProvingSimulated - - var proofSubmitted: Future[void] = newFuture[void]("proofSubmitted") - var subscription: Subscription - - setup: - clock = MockClock.new() - - proc onProofSubmission(id: SlotId) = - proofSubmitted.complete() - proofSubmitted = newFuture[void]("proofSubmitted") - - market = MockMarket.new() - market.slotState[slot.id] = SlotState.Filled - market.setProofRequired(slot.id, true) - subscription = await market.subscribeProofSubmission(onProofSubmission) - - let onProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - return success(proof) - let context = SalesContext(market: market, clock: clock, onProve: onProve.some) - agent = newSalesAgent(context, request.id, slot.slotIndex, request.some) - state = SaleProvingSimulated.new() - state.failEveryNProofs = failEveryNProofs - - teardown: - await subscription.unsubscribe() - - proc advanceToNextPeriod(market: Market) {.async.} = - let periodicity = await market.periodicity() - let current = periodicity.periodOf(clock.now().Timestamp) - let periodEnd = periodicity.periodEnd(current) - clock.set(periodEnd.toSecondsSince1970 + 1) - - proc waitForProvingRounds(market: Market, rounds: int) {.async.} = - var rnds = rounds - 1 # proof round runs prior to advancing - while rnds > 0: - await market.advanceToNextPeriod() - await proofSubmitted - rnds -= 1 - - test "switches to cancelled state when request expires": - let next = state.onCancelled(request) - check !next of SaleCancelled - - test "switches to failed state when request fails": - let next = state.onFailed(request) - check !next of SaleFailed - - test "submits invalid proof every 3 proofs": - let future = state.run(agent) - let invalid = Groth16Proof.default - - await market.waitForProvingRounds(totalProofs) - check market.submitted == @[proof, proof, invalid, proof, proof, invalid] - - await future.cancelAndWait() - - test "switches to payout state when request is finished": - market.slotState[slot.id] = SlotState.Filled - - let future = state.run(agent) - - market.slotState[slot.id] = SlotState.Finished - await market.advanceToNextPeriod() - - check eventually future.finished - check !(future.read()) of SalePayout diff --git a/tests/codex/sales/states/testslotreserving.nim b/tests/codex/sales/states/testslotreserving.nim deleted file mode 100644 index b223338a..00000000 --- a/tests/codex/sales/states/testslotreserving.nim +++ /dev/null @@ -1,69 +0,0 @@ -import pkg/chronos -import pkg/questionable -import pkg/codex/contracts/requests -import pkg/codex/sales/states/slotreserving -import pkg/codex/sales/states/downloading -import pkg/codex/sales/states/cancelled -import pkg/codex/sales/states/failed -import pkg/codex/sales/states/ignored -import pkg/codex/sales/states/errored -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/sales/reservations -import pkg/codex/stores/repostore -import ../../../asynctest -import ../../helpers -import ../../examples -import ../../helpers/mockmarket -import ../../helpers/mockclock - -asyncchecksuite "sales state 'SlotReserving'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - var market: MockMarket - var clock: MockClock - var agent: SalesAgent - var state: SaleSlotReserving - var context: SalesContext - - setup: - market = MockMarket.new() - clock = MockClock.new() - - state = SaleSlotReserving.new() - context = SalesContext(market: market, clock: clock) - - agent = newSalesAgent(context, request.id, slotIndex, request.some) - - test "switches to cancelled state when request expires": - let next = state.onCancelled(request) - check !next of SaleCancelled - - test "switches to failed state when request fails": - let next = state.onFailed(request) - check !next of SaleFailed - - test "run switches to downloading when slot successfully reserved": - let next = await state.run(agent) - check !next of SaleDownloading - - test "run switches to ignored when slot reservation not allowed": - market.setCanReserveSlot(false) - let next = await state.run(agent) - check !next of SaleIgnored - - test "run switches to errored when slot reservation errors": - let error = newException(MarketError, "some error") - market.setErrorOnReserveSlot(error) - let next = !(await state.run(agent)) - check next of SaleErrored - let errored = SaleErrored(next) - check errored.error == error - - test "run switches to ignored when reservation is not allowed": - let error = - newException(SlotReservationNotAllowedError, "Reservation is not allowed") - market.setErrorOnReserveSlot(error) - let next = !(await state.run(agent)) - check next of SaleIgnored - check SaleIgnored(next).reprocessSlot == false diff --git a/tests/codex/sales/states/testunknown.nim b/tests/codex/sales/states/testunknown.nim deleted file mode 100644 index 4806122f..00000000 --- a/tests/codex/sales/states/testunknown.nim +++ /dev/null @@ -1,67 +0,0 @@ -import pkg/codex/contracts/requests -import pkg/codex/sales -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/sales/states/unknown -import pkg/codex/sales/states/errored -import pkg/codex/sales/states/filled -import pkg/codex/sales/states/finished -import pkg/codex/sales/states/failed -import pkg/codex/sales/states/payout - -import ../../../asynctest -import ../../helpers/mockmarket -import ../../examples -import ../../helpers - -suite "sales state 'unknown'": - let request = StorageRequest.example - let slotIndex = request.ask.slots div 2 - let slotId = slotId(request.id, slotIndex) - - var market: MockMarket - var context: SalesContext - var agent: SalesAgent - var state: SaleUnknown - - setup: - market = MockMarket.new() - context = SalesContext(market: market) - agent = newSalesAgent(context, request.id, slotIndex, request.some) - state = SaleUnknown.new() - - test "switches to error state when the request cannot be retrieved": - agent = newSalesAgent(context, request.id, slotIndex, StorageRequest.none) - let next = await state.run(agent) - check !next of SaleErrored - check SaleErrored(!next).error.msg == "request could not be retrieved" - - test "switches to error state when on chain state cannot be fetched": - let next = await state.run(agent) - check !next of SaleErrored - - test "switches to error state when on chain state is 'free'": - market.slotState[slotId] = SlotState.Free - let next = await state.run(agent) - check !next of SaleErrored - check SaleErrored(!next).error.msg == "Slot state on chain should not be 'free'" - - test "switches to filled state when on chain state is 'filled'": - market.slotState[slotId] = SlotState.Filled - let next = await state.run(agent) - check !next of SaleFilled - - test "switches to payout state when on chain state is 'finished'": - market.slotState[slotId] = SlotState.Finished - let next = await state.run(agent) - check !next of SalePayout - - test "switches to finished state when on chain state is 'paid'": - market.slotState[slotId] = SlotState.Paid - let next = await state.run(agent) - check !next of SaleFinished - - test "switches to failed state when on chain state is 'failed'": - market.slotState[slotId] = SlotState.Failed - let next = await state.run(agent) - check !next of SaleFailed diff --git a/tests/codex/sales/testreservations.nim b/tests/codex/sales/testreservations.nim deleted file mode 100644 index 7d958ee3..00000000 --- a/tests/codex/sales/testreservations.nim +++ /dev/null @@ -1,538 +0,0 @@ -import std/random -import std/times -import pkg/questionable -import pkg/questionable/results -import pkg/chronos -import pkg/datastore - -import pkg/codex/stores -import pkg/codex/errors -import pkg/codex/sales -import pkg/codex/clock -import pkg/codex/utils/json - -import ../../asynctest -import ../examples -import ../helpers - -const CONCURRENCY_TESTS_COUNT = 1000 - -asyncchecksuite "Reservations module": - var - repo: RepoStore - repoDs: Datastore - metaDs: Datastore - reservations: Reservations - collateralPerByte: UInt256 - let - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - - setup: - randomize(1.int64) # create reproducible results - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() - repo = RepoStore.new(repoDs, metaDs) - reservations = Reservations.new(repo) - collateralPerByte = uint8.example.u256 - - teardown: - await repoTmp.destroyDb() - await metaTmp.destroyDb() - - proc createAvailability(enabled = true, until = 0.SecondsSince1970): Availability = - let example = Availability.example(collateralPerByte) - let totalSize = rand(100000 .. 200000).uint64 - let totalCollateral = totalSize.u256 * collateralPerByte - let availability = waitFor reservations.createAvailability( - totalSize, example.duration, example.minPricePerBytePerSecond, totalCollateral, - enabled, until, - ) - return availability.get - - proc createReservation(availability: Availability): Reservation = - let size = rand(1 ..< availability.freeSize.int) - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let reservation = waitFor reservations.createReservation( - availability.id, size.uint64, RequestId.example, uint64.example, 1.u256, - validUntil, - ) - return reservation.get - - test "availability can be serialised and deserialised": - let availability = Availability.example - let serialised = %availability - check Availability.fromJson(serialised).get == availability - - test "has no availability initially": - check (await reservations.all(Availability)).get.len == 0 - - test "generates unique ids for storage availability": - let availability1 = Availability.init( - 1.uint64, 2.uint64, 3.uint64, 4.u256, 5.u256, true, 0.SecondsSince1970 - ) - let availability2 = Availability.init( - 1.uint64, 2.uint64, 3.uint64, 4.u256, 5.u256, true, 0.SecondsSince1970 - ) - check availability1.id != availability2.id - - test "can reserve available storage": - let availability = createAvailability() - check availability.id != AvailabilityId.default - - test "creating availability reserves bytes in repo": - let orig = repo.available.uint - let availability = createAvailability() - check repo.available.uint == orig - availability.freeSize - - test "can get all availabilities": - let availability1 = createAvailability() - let availability2 = createAvailability() - let availabilities = !(await reservations.all(Availability)) - check: - # perform unordered checks - availabilities.len == 2 - availabilities.contains(availability1) - availabilities.contains(availability2) - - test "reserved availability exists": - let availability = createAvailability() - - let exists = await reservations.exists(availability.key.get) - - check exists - - test "reservation can be created": - let availability = createAvailability() - let reservation = createReservation(availability) - check reservation.id != ReservationId.default - - test "can get all reservations": - let availability1 = createAvailability() - let availability2 = createAvailability() - let reservation1 = createReservation(availability1) - let reservation2 = createReservation(availability2) - let availabilities = !(await reservations.all(Availability)) - let reservations = !(await reservations.all(Reservation)) - check: - # perform unordered checks - availabilities.len == 2 - reservations.len == 2 - reservations.contains(reservation1) - reservations.contains(reservation2) - - test "can get reservations of specific availability": - let availability1 = createAvailability() - let availability2 = createAvailability() - let reservation1 = createReservation(availability1) - let reservation2 = createReservation(availability2) - let reservations = !(await reservations.all(Reservation, availability1.id)) - - check: - # perform unordered checks - reservations.len == 1 - reservations.contains(reservation1) - not reservations.contains(reservation2) - - test "cannot create reservation with non-existant availability": - let availability = Availability.example - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let created = await reservations.createReservation( - availability.id, uint64.example, RequestId.example, uint64.example, 1.u256, - validUntil, - ) - check created.isErr - check created.error of NotExistsError - - test "cannot create reservation larger than availability size": - let availability = createAvailability() - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let created = await reservations.createReservation( - availability.id, - availability.totalSize + 1, - RequestId.example, - uint64.example, - UInt256.example, - validUntil, - ) - check created.isErr - check created.error of BytesOutOfBoundsError - - test "cannot create reservation larger than availability size - concurrency test": - proc concurrencyTest(): Future[void] {.async.} = - let availability = createAvailability() - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let one = reservations.createReservation( - availability.id, - availability.totalSize - 1, - RequestId.example, - uint64.example, - UInt256.example, - validUntil, - ) - - let two = reservations.createReservation( - availability.id, availability.totalSize, RequestId.example, uint64.example, - UInt256.example, validUntil, - ) - - let oneResult = await one - let twoResult = await two - - check oneResult.isErr or twoResult.isErr - - if oneResult.isErr: - check oneResult.error of BytesOutOfBoundsError - if twoResult.isErr: - check twoResult.error of BytesOutOfBoundsError - - var futures: seq[Future[void]] - for _ in 1 .. CONCURRENCY_TESTS_COUNT: - futures.add(concurrencyTest()) - - await allFuturesThrowing(futures) - - test "creating reservation reduces availability size": - let availability = createAvailability() - let orig = availability.freeSize - let reservation = createReservation(availability) - let key = availability.id.key.get - let updated = (await reservations.get(key, Availability)).get - check updated.freeSize == orig - reservation.size - - test "can check if reservation exists": - let availability = createAvailability() - let reservation = createReservation(availability) - let key = reservation.key.get - check await reservations.exists(key) - - test "non-existant availability does not exist": - let key = AvailabilityId.example.key.get - check not (await reservations.exists(key)) - - test "non-existant reservation does not exist": - let key = key(ReservationId.example, AvailabilityId.example).get - check not (await reservations.exists(key)) - - test "can check if availability exists": - let availability = createAvailability() - let key = availability.key.get - check await reservations.exists(key) - - test "can delete reservation": - let availability = createAvailability() - let reservation = createReservation(availability) - check isOk ( - await reservations.deleteReservation(reservation.id, reservation.availabilityId) - ) - let key = reservation.key.get - check not (await reservations.exists(key)) - - test "deleting reservation returns bytes back to availability": - let availability = createAvailability() - let orig = availability.freeSize - let reservation = createReservation(availability) - discard - await reservations.deleteReservation(reservation.id, reservation.availabilityId) - let key = availability.key.get - let updated = !(await reservations.get(key, Availability)) - check updated.freeSize == orig - - test "calling returnBytesToAvailability returns bytes back to availability": - let availability = createAvailability() - let reservation = createReservation(availability) - let orig = availability.freeSize - reservation.size - let origQuota = repo.quotaReservedBytes - let returnedBytes = reservation.size + 200.uint64 - - check isOk await reservations.returnBytesToAvailability( - reservation.availabilityId, reservation.id, returnedBytes - ) - - let key = availability.key.get - let updated = !(await reservations.get(key, Availability)) - - check updated.freeSize > orig - check (updated.freeSize - orig) == 200.uint64 - check (repo.quotaReservedBytes - origQuota) == 200.NBytes - - test "update releases quota when lowering size": - let - availability = createAvailability() - origQuota = repo.quotaReservedBytes - availability.totalSize = availability.totalSize - 100 - - check isOk await reservations.update(availability) - check (origQuota - repo.quotaReservedBytes) == 100.NBytes - - test "update reserves quota when growing size": - let - availability = createAvailability() - origQuota = repo.quotaReservedBytes - availability.totalSize = availability.totalSize + 100 - - check isOk await reservations.update(availability) - check (repo.quotaReservedBytes - origQuota) == 100.NBytes - - test "create availability set enabled to true by default": - let availability = createAvailability() - check availability.enabled == true - - test "create availability set until to 0 by default": - let availability = createAvailability() - check availability.until == 0.SecondsSince1970 - - test "create availability whith correct values": - var until = getTime().toUnix() - - let availability = createAvailability(enabled = false, until = until) - check availability.enabled == false - check availability.until == until - - test "create an availability fails when trying set until with a negative value": - let totalSize = rand(100000 .. 200000).uint64 - let example = Availability.example(collateralPerByte) - let totalCollateral = totalSize.u256 * collateralPerByte - - let result = await reservations.createAvailability( - totalSize, - example.duration, - example.minPricePerBytePerSecond, - totalCollateral, - enabled = true, - until = -1.SecondsSince1970, - ) - - check result.isErr - check result.error of UntilOutOfBoundsError - - test "update an availability fails when trying set until with a negative value": - let until = getTime().toUnix() - let availability = createAvailability(until = until) - - availability.until = -1 - - let result = await reservations.update(availability) - check result.isErr - check result.error of UntilOutOfBoundsError - - test "reservation can be partially released": - let availability = createAvailability() - let reservation = createReservation(availability) - check isOk await reservations.release(reservation.id, reservation.availabilityId, 1) - let key = reservation.key.get - let updated = !(await reservations.get(key, Reservation)) - check updated.size == reservation.size - 1 - - test "cannot release more bytes than size of reservation": - let availability = createAvailability() - let reservation = createReservation(availability) - let updated = await reservations.release( - reservation.id, reservation.availabilityId, reservation.size + 1 - ) - check updated.isErr - check updated.error of BytesOutOfBoundsError - - test "cannot release bytes from non-existant reservation": - let availability = createAvailability() - discard createReservation(availability) - let updated = await reservations.release(ReservationId.example, availability.id, 1) - check updated.isErr - check updated.error of NotExistsError - - test "OnAvailabilitySaved called when availability is created": - var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - added = a - - let availability = createAvailability() - - check added == availability - - test "OnAvailabilitySaved called when availability size is increased": - var availability = createAvailability() - var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - added = a - availability.freeSize += 1 - discard await reservations.update(availability) - - check added == availability - - test "OnAvailabilitySaved is not called when availability size is decreased": - var availability = createAvailability() - var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - called = true - availability.freeSize -= 1.uint64 - discard await reservations.update(availability) - - check not called - - test "OnAvailabilitySaved is not called when availability is disabled": - var availability = createAvailability(enabled = false) - var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - called = true - availability.freeSize -= 1 - discard await reservations.update(availability) - - check not called - - test "OnAvailabilitySaved called when availability duration is increased": - var availability = createAvailability() - var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - added = a - availability.duration += 1 - discard await reservations.update(availability) - - check added == availability - - test "OnAvailabilitySaved is not called when availability duration is decreased": - var availability = createAvailability() - var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - called = true - availability.duration -= 1 - discard await reservations.update(availability) - - check not called - - test "OnAvailabilitySaved called when availability minPricePerBytePerSecond is increased": - var availability = createAvailability() - var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - added = a - availability.minPricePerBytePerSecond += 1.u256 - discard await reservations.update(availability) - - check added == availability - - test "OnAvailabilitySaved is not called when availability minPricePerBytePerSecond is decreased": - var availability = createAvailability() - var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - called = true - availability.minPricePerBytePerSecond -= 1.u256 - discard await reservations.update(availability) - - check not called - - test "OnAvailabilitySaved called when availability totalRemainingCollateral is increased": - var availability = createAvailability() - var added: Availability - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - added = a - availability.totalRemainingCollateral = - availability.totalRemainingCollateral + 1.u256 - discard await reservations.update(availability) - - check added == availability - - test "OnAvailabilitySaved is not called when availability totalRemainingCollateral is decreased": - var availability = createAvailability() - var called = false - reservations.OnAvailabilitySaved = proc(a: Availability) {.async: (raises: []).} = - called = true - availability.totalRemainingCollateral = - availability.totalRemainingCollateral - 1.u256 - discard await reservations.update(availability) - - check not called - - test "availabilities can be found": - let availability = createAvailability() - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let found = await reservations.findAvailability( - availability.freeSize, availability.duration, - availability.minPricePerBytePerSecond, collateralPerByte, validUntil, - ) - - check found.isSome - check found.get == availability - - test "does not find an availability when is it disabled": - let availability = createAvailability(enabled = false) - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let found = await reservations.findAvailability( - availability.freeSize, availability.duration, - availability.minPricePerBytePerSecond, collateralPerByte, validUntil, - ) - - check found.isNone - - test "finds an availability when the until date is after the duration": - let example = Availability.example(collateralPerByte) - let until = getTime().toUnix() + example.duration.SecondsSince1970 - let availability = createAvailability(until = until) - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let found = await reservations.findAvailability( - availability.freeSize, availability.duration, - availability.minPricePerBytePerSecond, collateralPerByte, validUntil, - ) - - check found.isSome - check found.get == availability - - test "does not find an availability when the until date is before the duration": - let example = Availability.example(collateralPerByte) - let until = getTime().toUnix() + 1.SecondsSince1970 - let availability = createAvailability(until = until) - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let found = await reservations.findAvailability( - availability.freeSize, availability.duration, - availability.minPricePerBytePerSecond, collateralPerByte, validUntil, - ) - - check found.isNone - - test "non-matching availabilities are not found": - let availability = createAvailability() - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let found = await reservations.findAvailability( - availability.freeSize + 1, - availability.duration, - availability.minPricePerBytePerSecond, - collateralPerByte, - validUntil, - ) - - check found.isNone - - test "non-existent availability cannot be found": - let availability = Availability.example - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - let found = await reservations.findAvailability( - availability.freeSize, availability.duration, - availability.minPricePerBytePerSecond, collateralPerByte, validUntil, - ) - - check found.isNone - - test "non-existent availability cannot be retrieved": - let key = AvailabilityId.example.key.get - let got = await reservations.get(key, Availability) - check got.error of NotExistsError - - test "can get available bytes in repo": - check reservations.available == DefaultQuotaBytes.uint - - test "reports quota available to be reserved": - check reservations.hasAvailable(DefaultQuotaBytes.uint - 1) - - test "reports quota not available to be reserved": - check not reservations.hasAvailable(DefaultQuotaBytes.uint64 + 1) - - test "fails to create availability with size that is larger than available quota": - let created = await reservations.createAvailability( - DefaultQuotaBytes.uint64 + 1, - uint64.example, - UInt256.example, - UInt256.example, - enabled = true, - until = 0.SecondsSince1970, - ) - check created.isErr - check created.error of ReserveFailedError - check created.error.parent of QuotaNotEnoughError diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim deleted file mode 100644 index 3b0eec28..00000000 --- a/tests/codex/sales/testsales.nim +++ /dev/null @@ -1,706 +0,0 @@ -import std/sequtils -import std/sugar -import std/times -import pkg/chronos -import pkg/datastore/typedds -import pkg/questionable -import pkg/questionable/results -import pkg/codex/sales -import pkg/codex/sales/salesdata -import pkg/codex/sales/salescontext -import pkg/codex/sales/reservations -import pkg/codex/sales/slotqueue -import pkg/codex/stores/repostore -import pkg/codex/blocktype as bt -import pkg/codex/node -import pkg/codex/utils/asyncstatemachine -import times -import ../../asynctest -import ../helpers -import ../helpers/mockmarket -import ../helpers/mockclock -import ../helpers/always -import ../examples -import ./helpers/periods - -asyncchecksuite "Sales - start": - let - proof = Groth16Proof.example - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - - var request: StorageRequest - var sales: Sales - var market: MockMarket - var clock: MockClock - var reservations: Reservations - var repo: RepoStore - var queue: SlotQueue - var itemsProcessed: seq[SlotQueueItem] - - setup: - request = StorageRequest( - ask: StorageAsk( - slots: 4, - slotSize: 100.uint64, - duration: 60.uint64, - pricePerBytePerSecond: 1.u256, - collateralPerByte: 1.u256, - ), - content: StorageContent( - cid: Cid.init("zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob").tryGet - ), - expiry: 60, - ) - - market = MockMarket.new() - clock = MockClock.new() - let repoDs = repoTmp.newDb() - let metaDs = metaTmp.newDb() - repo = RepoStore.new(repoDs, metaDs) - await repo.start() - sales = Sales.new(market, clock, repo) - reservations = sales.context.reservations - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - return success() - - sales.onExpiryUpdate = proc( - rootCid: Cid, expiry: SecondsSince1970 - ): Future[?!void] {.async: (raises: [CancelledError]).} = - return success() - - queue = sales.context.slotQueue - sales.onProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - return success(proof) - itemsProcessed = @[] - - teardown: - await sales.stop() - await repo.stop() - await repoTmp.destroyDb() - await metaTmp.destroyDb() - - proc fillSlot(slotIdx: uint64 = 0.uint64) {.async.} = - let address = await market.getSigner() - let slot = - MockSlot(requestId: request.id, slotIndex: slotIdx, proof: proof, host: address) - market.filled.add slot - market.slotState[slotId(request.id, slotIdx)] = SlotState.Filled - - test "load slots when Sales module starts": - let me = await market.getSigner() - - request.ask.slots = 2 - market.requested = @[request] - market.requestState[request.id] = RequestState.New - - let slot0 = MockSlot(requestId: request.id, slotIndex: 0, proof: proof, host: me) - await fillSlot(slot0.slotIndex) - - let slot1 = MockSlot(requestId: request.id, slotIndex: 1, proof: proof, host: me) - await fillSlot(slot1.slotIndex) - - market.activeSlots[me] = @[request.slotId(0), request.slotId(1)] - market.requested = @[request] - market.activeRequests[me] = @[request.id] - - await sales.start() - - check eventually sales.agents.len == 2 - check sales.agents.any( - agent => agent.data.requestId == request.id and agent.data.slotIndex == 0.uint64 - ) - check sales.agents.any( - agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.uint64 - ) - -asyncchecksuite "Sales": - let - proof = Groth16Proof.example - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - - var totalAvailabilitySize: uint64 - var minPricePerBytePerSecond: UInt256 - var requestedCollateralPerByte: UInt256 - var totalCollateral: UInt256 - var availability: Availability - var request: StorageRequest - var sales: Sales - var market: MockMarket - var clock: MockClock - var reservations: Reservations - var repo: RepoStore - var queue: SlotQueue - var itemsProcessed: seq[SlotQueueItem] - - setup: - totalAvailabilitySize = 100.uint64 - minPricePerBytePerSecond = 1.u256 - requestedCollateralPerByte = 1.u256 - totalCollateral = requestedCollateralPerByte * totalAvailabilitySize.stuint(256) - availability = Availability.init( - totalSize = totalAvailabilitySize, - freeSize = totalAvailabilitySize, - duration = 60.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - enabled = true, - until = 0.SecondsSince1970, - ) - request = StorageRequest( - ask: StorageAsk( - slots: 4, - slotSize: 100.uint64, - duration: 60.uint64, - pricePerBytePerSecond: minPricePerBytePerSecond, - collateralPerByte: 1.u256, - ), - content: StorageContent( - cid: Cid.init("zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob").tryGet - ), - expiry: 60, - ) - - market = MockMarket.new() - - let me = await market.getSigner() - market.activeSlots[me] = @[] - - clock = MockClock.new() - let repoDs = repoTmp.newDb() - let metaDs = metaTmp.newDb() - repo = RepoStore.new(repoDs, metaDs) - await repo.start() - sales = Sales.new(market, clock, repo) - reservations = sales.context.reservations - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - return success() - - sales.onExpiryUpdate = proc( - rootCid: Cid, expiry: SecondsSince1970 - ): Future[?!void] {.async: (raises: [CancelledError]).} = - return success() - - queue = sales.context.slotQueue - sales.onProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - return success(proof) - await sales.start() - itemsProcessed = @[] - - teardown: - await sales.stop() - await repo.stop() - await repoTmp.destroyDb() - await metaTmp.destroyDb() - - proc isInState(idx: int, state: string): bool = - proc description(state: State): string = - $state - - if idx >= sales.agents.len: - return false - sales.agents[idx].query(description) == state.some - - proc allowRequestToStart() {.async.} = - check eventually isInState(0, "SaleInitialProving") - # it won't start proving until the next period - await clock.advanceToNextPeriod(market) - - proc getAvailability(): Availability = - let key = availability.id.key.get - (waitFor reservations.get(key, Availability)).get - - proc createAvailability(enabled = true, until = 0.SecondsSince1970) = - let a = waitFor reservations.createAvailability( - availability.totalSize, availability.duration, - availability.minPricePerBytePerSecond, availability.totalCollateral, enabled, - until, - ) - availability = a.get # update id - - proc notProcessed(itemsProcessed: seq[SlotQueueItem], request: StorageRequest): bool = - let items = SlotQueueItem.init(request, collateral = request.ask.collateralPerSlot) - for i in 0 ..< items.len: - if itemsProcessed.contains(items[i]): - return false - return true - - proc addRequestToSaturatedQueue(): Future[StorageRequest] {.async.} = - queue.onProcessSlot = proc(item: SlotQueueItem) {.async: (raises: []).} = - try: - await sleepAsync(10.millis) - itemsProcessed.add item - except CancelledError as exc: - checkpoint(exc.msg) - - var request1 = StorageRequest.example - request1.ask.collateralPerByte = request.ask.collateralPerByte + 1 - createAvailability() - # saturate queue - while queue.len < queue.size - 1: - await market.requestStorage(StorageRequest.example) - # send request - await market.requestStorage(request1) - await sleepAsync(5.millis) # wait for request slots to be added to queue - return request1 - - proc wasIgnored(): bool = - let run = proc(): Future[bool] {.async.} = - always ( - getAvailability().freeSize == availability.freeSize and - (waitFor reservations.all(Reservation)).get.len == 0 - ) - waitFor run() - - test "processes all request's slots once StorageRequested emitted": - queue.onProcessSlot = proc(item: SlotQueueItem) {.async: (raises: []).} = - itemsProcessed.add item - createAvailability() - await market.requestStorage(request) - let items = SlotQueueItem.init(request, collateral = request.ask.collateralPerSlot) - check eventually items.allIt(itemsProcessed.contains(it)) - - test "removes slots from slot queue once RequestCancelled emitted": - let request1 = await addRequestToSaturatedQueue() - market.emitRequestCancelled(request1.id) - check always itemsProcessed.notProcessed(request1) - - test "removes request from slot queue once RequestFailed emitted": - let request1 = await addRequestToSaturatedQueue() - market.emitRequestFailed(request1.id) - check always itemsProcessed.notProcessed(request1) - - test "removes request from slot queue once RequestFulfilled emitted": - let request1 = await addRequestToSaturatedQueue() - market.emitRequestFulfilled(request1.id) - check always itemsProcessed.notProcessed(request1) - - test "removes slot index from slot queue once SlotFilled emitted": - let request1 = await addRequestToSaturatedQueue() - market.emitSlotFilled(request1.id, 1.uint64) - let expected = - SlotQueueItem.init(request1, 1'u16, collateral = request1.ask.collateralPerSlot) - check always (not itemsProcessed.contains(expected)) - - test "removes slot index from slot queue once SlotReservationsFull emitted": - let request1 = await addRequestToSaturatedQueue() - market.emitSlotReservationsFull(request1.id, 1.uint64) - let expected = - SlotQueueItem.init(request1, 1'u16, collateral = request1.ask.collateralPerSlot) - check always (not itemsProcessed.contains(expected)) - - test "adds slot index to slot queue once SlotFreed emitted": - queue.onProcessSlot = proc(item: SlotQueueItem) {.async: (raises: []).} = - itemsProcessed.add item - - createAvailability() - market.requested.add request # "contract" must be able to return request - - market.emitSlotFreed(request.id, 2.uint64) - - without collateralPerSlot =? await market.slotCollateral(request.id, 2.uint64), - error: - fail() - - let expected = - SlotQueueItem.init(request, 2.uint16, collateral = request.ask.collateralPerSlot) - - check eventually itemsProcessed.contains(expected) - - test "items in queue are readded (and marked seen) once ignored": - await market.requestStorage(request) - let items = SlotQueueItem.init(request, collateral = request.ask.collateralPerSlot) - check eventually queue.len > 0 - # queue starts paused, allow items to be added to the queue - check eventually queue.paused - # The first processed item will be will have been re-pushed with `seen = - # true`. Then, once this item is processed by the queue, its 'seen' flag - # will be checked, at which point the queue will be paused. This test could - # check item existence in the queue, but that would require inspecting - # onProcessSlot to see which item was first, and overridding onProcessSlot - # will prevent the queue working as expected in the Sales module. - check eventually queue.len == 4 - - for item in items: - check queue.contains(item) - - for i in 0 ..< queue.len: - check queue[i].seen - - test "queue is paused once availability is insufficient to service slots in queue": - createAvailability() # enough to fill a single slot - await market.requestStorage(request) - let items = SlotQueueItem.init(request, collateral = request.ask.collateralPerSlot) - check eventually queue.len > 0 - # queue starts paused, allow items to be added to the queue - check eventually queue.paused - # The first processed item/slot will be filled (eventually). Subsequent - # items will be processed and eventually re-pushed with `seen = true`. Once - # a "seen" item is processed by the queue, the queue is paused. In the - # meantime, the other items that are process, marked as seen, and re-added - # to the queue may be processed simultaneously as the queue pausing. - # Therefore, there should eventually be 3 items remaining in the queue, all - # seen. - check eventually queue.len == 3 - for i in 0 ..< queue.len: - check queue[i].seen - - test "availability size is reduced by request slot size when fully downloaded": - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - let blk = bt.Block.new(@[1.byte]).get - await onBatch(blk.repeat(request.ask.slotSize.int)) - - createAvailability() - await market.requestStorage(request) - check eventually getAvailability().freeSize == - availability.freeSize - request.ask.slotSize - - test "bytes are returned to availability once finished": - var slotIndex = 0.uint64 - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - slotIndex = slot - let blk = bt.Block.new(@[1.byte]).get - await onBatch(blk.repeat(request.ask.slotSize)) - - let sold = newFuture[void]() - sales.onSale = proc(request: StorageRequest, slotIndex: uint64) = - sold.complete() - - createAvailability() - let origSize = availability.freeSize - await market.requestStorage(request) - await allowRequestToStart() - await sold - - # Disable the availability; otherwise, it will pick up the - # reservation again and we will not be able to check - # if the bytes are returned - availability.enabled = false - let result = await reservations.update(availability) - check result.isOk - - # complete request - market.slotState[request.slotId(slotIndex)] = SlotState.Finished - clock.advance(request.ask.duration.int64) - - check eventually getAvailability().freeSize == origSize - - test "ignores download when duration not long enough": - availability.duration = request.ask.duration - 1 - createAvailability() - await market.requestStorage(request) - check wasIgnored() - - test "ignores request when slot size is too small": - availability.totalSize = request.ask.slotSize - 1 - createAvailability() - await market.requestStorage(request) - check wasIgnored() - - test "ignores request when reward is too low": - availability.minPricePerBytePerSecond = request.ask.pricePerBytePerSecond + 1 - createAvailability() - await market.requestStorage(request) - check wasIgnored() - - test "ignores request when asked collateral is too high": - var tooBigCollateral = request - tooBigCollateral.ask.collateralPerByte = requestedCollateralPerByte + 1 - createAvailability() - await market.requestStorage(tooBigCollateral) - check wasIgnored() - - test "ignores request when slot state is not free": - createAvailability() - await market.requestStorage(request) - market.slotState[request.slotId(0.uint64)] = SlotState.Filled - market.slotState[request.slotId(1.uint64)] = SlotState.Filled - market.slotState[request.slotId(2.uint64)] = SlotState.Filled - market.slotState[request.slotId(3.uint64)] = SlotState.Filled - check wasIgnored() - - test "ignores request when availability is not enabled": - createAvailability(enabled = false) - await market.requestStorage(request) - check wasIgnored() - - test "ignores request when availability until terminates before the duration": - let until = getTime().toUnix() - createAvailability(until = until) - await market.requestStorage(request) - - check wasIgnored() - - test "retrieves request when availability until terminates after the duration": - let requestEnd = getTime().toUnix() + cast[int64](request.ask.duration) - let until = requestEnd + 1 - createAvailability(until = until) - - var storingRequest: StorageRequest - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - storingRequest = request - return success() - - market.requestEnds[request.id] = requestEnd - await market.requestStorage(request) - check eventually storingRequest == request - - test "retrieves and stores data locally": - var storingRequest: StorageRequest - var storingSlot: uint64 - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - storingRequest = request - storingSlot = slot - return success() - createAvailability() - await market.requestStorage(request) - check eventually storingRequest == request - check storingSlot < request.ask.slots - - test "makes storage available again when data retrieval fails": - let error = newException(IOError, "data retrieval failed") - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - return failure(error) - createAvailability() - await market.requestStorage(request) - check getAvailability().freeSize == availability.freeSize - - test "generates proof of storage": - var provingRequest: StorageRequest - var provingSlot: uint64 - sales.onProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - provingRequest = slot.request - provingSlot = slot.slotIndex - return success(Groth16Proof.example) - createAvailability() - await market.requestStorage(request) - await allowRequestToStart() - - check eventually provingRequest == request - check provingSlot < request.ask.slots - - test "fills a slot": - createAvailability() - await market.requestStorage(request) - await allowRequestToStart() - - check eventually market.filled.len > 0 - check market.filled[0].requestId == request.id - check market.filled[0].slotIndex < request.ask.slots - check market.filled[0].proof == proof - check market.filled[0].host == await market.getSigner() - - test "calls onFilled when slot is filled": - var soldRequest = StorageRequest.default - var soldSlotIndex = uint64.high - sales.onSale = proc(request: StorageRequest, slotIndex: uint64) = - soldRequest = request - soldSlotIndex = slotIndex - createAvailability() - await market.requestStorage(request) - await allowRequestToStart() - - check eventually soldRequest == request - check soldSlotIndex < request.ask.slots - - test "calls onClear when storage becomes available again": - # fail the proof intentionally to trigger `agent.finish(success=false)`, - # which then calls the onClear callback - sales.onProve = proc( - slot: Slot, challenge: ProofChallenge - ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = - return failure("proof failed") - var clearedRequest: StorageRequest - var clearedSlotIndex: uint64 - sales.onClear = proc(request: StorageRequest, slotIndex: uint64) = - clearedRequest = request - clearedSlotIndex = slotIndex - createAvailability() - await market.requestStorage(request) - await allowRequestToStart() - - check eventually clearedRequest == request - check clearedSlotIndex < request.ask.slots - - test "makes storage available again when other host fills the slot": - let otherHost = Address.example - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - await sleepAsync(chronos.hours(1)) - return success() - createAvailability() - await market.requestStorage(request) - for slotIndex in 0 ..< request.ask.slots: - market.fillSlot(request.id, slotIndex.uint64, proof, otherHost) - check eventually (await reservations.all(Availability)).get == @[availability] - - test "makes storage available again when request expires": - let origSize = availability.freeSize - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - await sleepAsync(chronos.hours(1)) - return success() - createAvailability() - await market.requestStorage(request) - - # If we would not await, then the `clock.set` would run "too fast" as the `subscribeCancellation()` - # would otherwise not set the timeout early enough as it uses `clock.now` in the deadline calculation. - await sleepAsync(chronos.milliseconds(100)) - market.requestState[request.id] = RequestState.Cancelled - clock.set(market.requestExpiry[request.id] + 1) - check eventually (await reservations.all(Availability)).get == @[availability] - check getAvailability().freeSize == origSize - - test "verifies that request is indeed expired from onchain before firing onCancelled": - # ensure only one slot, otherwise once bytes are returned to the - # availability, the queue will be unpaused and availability will be consumed - # by other slots - request.ask.slots = 1 - market.requestEnds[request.id] = - getTime().toUnix() + cast[int64](request.ask.duration) - - let origSize = availability.freeSize - sales.onStore = proc( - request: StorageRequest, - expiry: SecondsSince1970, - slot: uint64, - onBatch: BatchProc, - isRepairing = false, - ): Future[?!void] {.async: (raises: [CancelledError]).} = - await sleepAsync(chronos.hours(1)) - return success() - createAvailability() - await market.requestStorage(request) - market.requestState[request.id] = RequestState.New - # "On-chain" is the request still ongoing even after local expiration - - # If we would not await, then the `clock.set` would run "too fast" as the `subscribeCancellation()` - # would otherwise not set the timeout early enough as it uses `clock.now` in the deadline calculation. - await sleepAsync(chronos.milliseconds(100)) - clock.set(market.requestExpiry[request.id] + 1) - check getAvailability().freeSize == 0 - - market.requestState[request.id] = RequestState.Cancelled - # Now "on-chain" is also expired - check eventually getAvailability().freeSize == origSize - - test "loads active slots from market": - let me = await market.getSigner() - - request.ask.slots = 2 - market.requested = @[request] - market.requestState[request.id] = RequestState.New - market.requestEnds[request.id] = request.expiry.toSecondsSince1970 - - proc fillSlot(slotIdx: uint64 = 0) {.async.} = - let address = await market.getSigner() - let slot = - MockSlot(requestId: request.id, slotIndex: slotIdx, proof: proof, host: address) - market.filled.add slot - market.slotState[slotId(request.id, slotIdx)] = SlotState.Filled - - let slot0 = MockSlot(requestId: request.id, slotIndex: 0, proof: proof, host: me) - await fillSlot(slot0.slotIndex) - - let slot1 = MockSlot(requestId: request.id, slotIndex: 1, proof: proof, host: me) - await fillSlot(slot1.slotIndex) - market.activeSlots[me] = @[request.slotId(0), request.slotId(1)] - market.requested = @[request] - market.activeRequests[me] = @[request.id] - - await sales.load() - - check eventually sales.agents.len == 2 - check sales.agents.any( - agent => agent.data.requestId == request.id and agent.data.slotIndex == 0.uint64 - ) - check sales.agents.any( - agent => agent.data.requestId == request.id and agent.data.slotIndex == 1.uint64 - ) - - test "deletes inactive reservations on load": - createAvailability() - let validUntil = getTime().toUnix() + 30.SecondsSince1970 - discard await reservations.createReservation( - availability.id, 100.uint64, RequestId.example, 0.uint64, UInt256.example, - validUntil, - ) - check (await reservations.all(Reservation)).get.len == 1 - await sales.load() - check (await reservations.all(Reservation)).get.len == 0 - check getAvailability().freeSize == availability.freeSize # was restored - - test "update an availability fails when trying change the until date before an existing reservation": - let until = getTime().toUnix() + 300.SecondsSince1970 - createAvailability(until = until) - - market.requestEnds[request.id] = - getTime().toUnix() + cast[int64](request.ask.duration) - - await market.requestStorage(request) - await allowRequestToStart() - - availability.until = getTime().toUnix() - - let result = await reservations.update(availability) - check result.isErr - check result.error of UntilOutOfBoundsError diff --git a/tests/codex/sales/testsalesagent.nim b/tests/codex/sales/testsalesagent.nim deleted file mode 100644 index c795904d..00000000 --- a/tests/codex/sales/testsalesagent.nim +++ /dev/null @@ -1,124 +0,0 @@ -import std/times -import pkg/chronos -import pkg/codex/sales -import pkg/codex/sales/salesagent -import pkg/codex/sales/salescontext -import pkg/codex/sales/statemachine - -import ../../asynctest -import ../helpers/mockmarket -import ../helpers/mockclock -import ../helpers -import ../examples - -var onCancelCalled = false -var onFailedCalled = false -var onSlotFilledCalled = false - -type MockState = ref object of SaleState - -method `$`*(state: MockState): string = - "MockState" - -method onCancelled*(state: MockState, request: StorageRequest): ?State = - onCancelCalled = true - -method onFailed*(state: MockState, request: StorageRequest): ?State = - onFailedCalled = true - -method onSlotFilled*( - state: MockState, requestId: RequestId, slotIndex: uint64 -): ?State = - onSlotFilledCalled = true - -asyncchecksuite "Sales agent": - let request = StorageRequest.example - var agent: SalesAgent - var context: SalesContext - var slotIndex: uint64 - var market: MockMarket - var clock: MockClock - - setup: - market = MockMarket.new() - market.requestExpiry[request.id] = getTime().toUnix() + request.expiry.int64 - clock = MockClock.new() - context = SalesContext(market: market, clock: clock) - slotIndex = 0.uint64 - onCancelCalled = false - onFailedCalled = false - onSlotFilledCalled = false - agent = newSalesAgent(context, request.id, slotIndex, some request) - - teardown: - await agent.stop() - - test "can retrieve request": - agent = newSalesAgent(context, request.id, slotIndex, none StorageRequest) - market.requested = @[request] - await agent.retrieveRequest() - check agent.data.request == some request - - test "subscribe assigns cancelled future": - await agent.subscribe() - check not agent.data.cancelled.isNil - - test "unsubscribe deassigns canceleld future": - await agent.subscribe() - await agent.unsubscribe() - check agent.data.cancelled.isNil - - test "subscribe can be called multiple times, without overwriting subscriptions/futures": - await agent.subscribe() - let cancelled = agent.data.cancelled - await agent.subscribe() - check cancelled == agent.data.cancelled - - test "unsubscribe can be called multiple times": - await agent.subscribe() - await agent.unsubscribe() - await agent.unsubscribe() - - test "current state onCancelled called when cancel emitted": - agent.start(MockState.new()) - await agent.subscribe() - market.requestState[request.id] = RequestState.Cancelled - clock.set(market.requestExpiry[request.id] + 1) - check eventually onCancelCalled - - for requestState in { - RequestState.New, RequestState.Started, RequestState.Finished, RequestState.Failed - }: - test "onCancelled is not called when request state is " & $requestState: - agent.start(MockState.new()) - await agent.subscribe() - market.requestState[request.id] = requestState - clock.set(market.requestExpiry[request.id] + 1) - await sleepAsync(100.millis) - check not onCancelCalled - - for requestState in {RequestState.Started, RequestState.Finished, RequestState.Failed}: - test "cancelled future is finished when request state is " & $requestState: - agent.start(MockState.new()) - await agent.subscribe() - market.requestState[request.id] = requestState - clock.set(market.requestExpiry[request.id] + 1) - check eventually agent.data.cancelled.finished - - test "cancelled future is finished (cancelled) when onFulfilled called": - agent.start(MockState.new()) - await agent.subscribe() - agent.onFulfilled(request.id) - # Note: futures that are cancelled, and do not re-raise the CancelledError - # will have a state of completed, not cancelled. - check eventually agent.data.cancelled.completed() - - test "current state onFailed called when onFailed called": - agent.start(MockState.new()) - agent.onFailed(request.id) - check eventually onFailedCalled - - test "current state onSlotFilled called when slot filled emitted": - agent.start(MockState.new()) - agent.onSlotFilled(request.id, slotIndex) - check eventually onSlotFilledCalled diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim deleted file mode 100644 index 37435ba5..00000000 --- a/tests/codex/sales/testslotqueue.nim +++ /dev/null @@ -1,596 +0,0 @@ -import std/sequtils -import pkg/chronos -import pkg/questionable -import pkg/questionable/results - -import pkg/codex/logutils -import pkg/codex/sales/slotqueue - -import ../../asynctest -import ../helpers -import ../helpers/mockmarket -import ../helpers/mockslotqueueitem -import ../examples - -suite "Slot queue start/stop": - var queue: SlotQueue - - setup: - queue = SlotQueue.new() - - teardown: - await queue.stop() - - test "starts out not running": - check not queue.running - - test "queue starts paused": - check queue.paused - - test "can call start multiple times, and when already running": - queue.start() - queue.start() - check queue.running - - test "can call stop when already stopped": - await queue.stop() - check not queue.running - - test "can call stop when running": - queue.start() - await queue.stop() - check not queue.running - - test "can call stop multiple times": - queue.start() - await queue.stop() - await queue.stop() - check not queue.running - -suite "Slot queue workers": - var queue: SlotQueue - - proc onProcessSlot(item: SlotQueueItem) {.async: (raises: []).} = - try: - await sleepAsync(1000.millis) - except CatchableError as exc: - checkpoint(exc.msg) - - setup: - let request = StorageRequest.example - queue = SlotQueue.new(maxSize = 5, maxWorkers = 3) - queue.onProcessSlot = onProcessSlot - - teardown: - await queue.stop() - - test "maxWorkers cannot be 0": - expect ValueError: - discard SlotQueue.new(maxSize = 1, maxWorkers = 0) - - test "maxWorkers cannot surpass maxSize": - expect ValueError: - discard SlotQueue.new(maxSize = 1, maxWorkers = 2) - -suite "Slot queue": - var onProcessSlotCalled = false - var onProcessSlotCalledWith: seq[(RequestId, uint16)] - var queue: SlotQueue - var paused: bool - - proc newSlotQueue(maxSize, maxWorkers: int, processSlotDelay = 1.millis) = - queue = SlotQueue.new(maxWorkers, maxSize.uint16) - queue.onProcessSlot = proc(item: SlotQueueItem) {.async: (raises: []).} = - try: - await sleepAsync(processSlotDelay) - except CatchableError as exc: - checkpoint(exc.msg) - finally: - onProcessSlotCalled = true - onProcessSlotCalledWith.add (item.requestId, item.slotIndex) - - queue.start() - - setup: - onProcessSlotCalled = false - onProcessSlotCalledWith = @[] - - teardown: - paused = false - - await queue.stop() - - test "starts out empty": - newSlotQueue(maxSize = 2, maxWorkers = 2) - check queue.len == 0 - check $queue == "[]" - - test "reports correct size": - newSlotQueue(maxSize = 2, maxWorkers = 2) - check queue.size == 2 - - test "correctly compares SlotQueueItems": - var requestA = StorageRequest.example - requestA.ask.duration = 1.uint64 - requestA.ask.pricePerBytePerSecond = 1.u256 - check requestA.ask.pricePerSlot == 1.u256 * requestA.ask.slotSize.u256 - requestA.ask.collateralPerByte = 100000.u256 - requestA.expiry = 1001.uint64 - - var requestB = StorageRequest.example - requestB.ask.duration = 100.uint64 - requestB.ask.pricePerBytePerSecond = 1000.u256 - check requestB.ask.pricePerSlot == 100000.u256 * requestB.ask.slotSize.u256 - requestB.ask.collateralPerByte = 1.u256 - requestB.expiry = 1000.uint64 - - let itemA = - SlotQueueItem.init(requestA, 0, collateral = requestA.ask.collateralPerSlot) - let itemB = - SlotQueueItem.init(requestB, 0, collateral = requestB.ask.collateralPerSlot) - check itemB < itemA # B higher priority than A - check itemA > itemB - - test "correct prioritizes SlotQueueItems based on 'seen'": - let request = StorageRequest.example - let itemA = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 1.uint64, - duration: 1.uint64, - pricePerBytePerSecond: 2.u256, # profitability is higher (good) - collateral: 1.u256, - expiry: 1.uint64, - seen: true, # seen (bad), more weight than profitability - ) - let itemB = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 1.uint64, - duration: 1.uint64, - pricePerBytePerSecond: 1.u256, # profitability is lower (bad) - collateral: 1.u256, - expiry: 1.uint64, - seen: false, # not seen (good) - ) - check itemB.toSlotQueueItem < itemA.toSlotQueueItem # B higher priority than A - check itemA.toSlotQueueItem > itemB.toSlotQueueItem - - test "correct prioritizes SlotQueueItems based on profitability": - let request = StorageRequest.example - let itemA = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 1.uint64, - duration: 1.uint64, - pricePerBytePerSecond: 1.u256, # reward is lower (bad) - collateral: 1.u256, # collateral is lower (good) - expiry: 1.uint64, - seen: false, - ) - let itemB = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 1.uint64, - duration: 1.uint64, - pricePerBytePerSecond: 2.u256, - # reward is higher (good), more weight than collateral - collateral: 2.u256, # collateral is higher (bad) - expiry: 1.uint64, - seen: false, - ) - - check itemB.toSlotQueueItem < itemA.toSlotQueueItem # < indicates higher priority - - test "correct prioritizes SlotQueueItems based on collateral": - let request = StorageRequest.example - let itemA = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 1.uint64, - duration: 1.uint64, - pricePerBytePerSecond: 1.u256, - collateral: 2.u256, # collateral is higher (bad) - expiry: 2.uint64, # expiry is longer (good) - seen: false, - ) - let itemB = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 1.uint64, - duration: 1.uint64, - pricePerBytePerSecond: 1.u256, - collateral: 1.u256, # collateral is lower (good), more weight than expiry - expiry: 1.uint64, # expiry is shorter (bad) - seen: false, - ) - - check itemB.toSlotQueueItem < itemA.toSlotQueueItem # < indicates higher priority - - test "correct prioritizes SlotQueueItems based on expiry": - let request = StorageRequest.example - let itemA = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 1.uint64, # slotSize is smaller (good) - duration: 1.uint64, - pricePerBytePerSecond: 1.u256, - collateral: 1.u256, - expiry: 1.uint64, # expiry is shorter (bad) - seen: false, - ) - let itemB = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 2.uint64, # slotSize is larger (bad) - duration: 1.uint64, - pricePerBytePerSecond: 1.u256, - collateral: 1.u256, - expiry: 2.uint64, # expiry is longer (good), more weight than slotSize - seen: false, - ) - - check itemB.toSlotQueueItem < itemA.toSlotQueueItem # < indicates higher priority - - test "correct prioritizes SlotQueueItems based on slotSize": - let request = StorageRequest.example - let itemA = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 2.uint64, # slotSize is larger (bad) - duration: 1.uint64, - pricePerBytePerSecond: 1.u256, - collateral: 1.u256, - expiry: 1.uint64, # expiry is shorter (bad) - seen: false, - ) - let itemB = MockSlotQueueItem( - requestId: request.id, - slotIndex: 0, - slotSize: 1.uint64, # slotSize is smaller (good) - duration: 1.uint64, - pricePerBytePerSecond: 1.u256, - collateral: 1.u256, - expiry: 1.uint64, - seen: false, - ) - - check itemA.toSlotQueueItem < itemB.toSlotQueueItem # < indicates higher priority - - test "expands available all possible slot indices on init": - let request = StorageRequest.example - let items = SlotQueueItem.init(request, collateral = request.ask.collateralPerSlot) - check items.len.uint64 == request.ask.slots - var checked = 0 - for slotIndex in 0'u16 ..< request.ask.slots.uint16: - check items.anyIt( - it == - SlotQueueItem.init( - request, slotIndex, collateral = request.ask.collateralPerSlot - ) - ) - inc checked - check checked == items.len - - test "can process items": - newSlotQueue(maxSize = 2, maxWorkers = 2) - let item1 = SlotQueueItem.example - let item2 = SlotQueueItem.example - check queue.push(item1).isOk - check queue.push(item2).isOk - check eventually onProcessSlotCalledWith == - @[(item1.requestId, item1.slotIndex), (item2.requestId, item2.slotIndex)] - - test "can push items past number of maxWorkers": - newSlotQueue(maxSize = 2, maxWorkers = 2) - let item0 = SlotQueueItem.example - let item1 = SlotQueueItem.example - let item2 = SlotQueueItem.example - let item3 = SlotQueueItem.example - let item4 = SlotQueueItem.example - check isOk queue.push(item0) - check isOk queue.push(item1) - check isOk queue.push(item2) - check isOk queue.push(item3) - check isOk queue.push(item4) - - test "can support uint16.high slots": - var request = StorageRequest.example - let maxUInt16 = uint16.high - let uint64Slots = uint64(maxUInt16) - request.ask.slots = uint64Slots - let items = SlotQueueItem.init( - request.id, request.ask, 0, collateral = request.ask.collateralPerSlot - ) - check items.len.uint16 == maxUInt16 - - test "cannot support greater than uint16.high slots": - var request = StorageRequest.example - let int32Slots = uint16.high.int32 + 1 - let uint64Slots = uint64(int32Slots) - request.ask.slots = uint64Slots - expect SlotsOutOfRangeError: - discard SlotQueueItem.init( - request.id, request.ask, 0, collateral = request.ask.collateralPerSlot - ) - - test "cannot push duplicate items": - newSlotQueue(maxSize = 6, maxWorkers = 1, processSlotDelay = 15.millis) - let item0 = SlotQueueItem.example - let item1 = SlotQueueItem.example - let item2 = SlotQueueItem.example - check isOk queue.push(item0) - check isOk queue.push(item1) - check queue.push(@[item2, item2, item2, item2]).error of SlotQueueItemExistsError - - test "can add items past max maxSize": - newSlotQueue(maxSize = 4, maxWorkers = 2, processSlotDelay = 10.millis) - let item1 = SlotQueueItem.example - let item2 = SlotQueueItem.example - let item3 = SlotQueueItem.example - let item4 = SlotQueueItem.example - check queue.push(item1).isOk - check queue.push(item2).isOk - check queue.push(item3).isOk - check queue.push(item4).isOk - check eventually onProcessSlotCalledWith.len == 4 - - test "can delete items": - newSlotQueue(maxSize = 6, maxWorkers = 2, processSlotDelay = 10.millis) - let item0 = SlotQueueItem.example - let item1 = SlotQueueItem.example - let item2 = SlotQueueItem.example - let item3 = SlotQueueItem.example - check queue.push(item0).isOk - check queue.push(item1).isOk - check queue.push(item2).isOk - check queue.push(item3).isOk - queue.delete(item3) - check not queue.contains(item3) - - test "can delete item by request id and slot id": - newSlotQueue(maxSize = 8, maxWorkers = 1, processSlotDelay = 10.millis) - let request0 = StorageRequest.example - var request1 = StorageRequest.example - request1.ask.collateralPerByte += 1.u256 - let items0 = - SlotQueueItem.init(request0, collateral = request0.ask.collateralPerSlot) - let items1 = - SlotQueueItem.init(request1, collateral = request1.ask.collateralPerSlot) - check queue.push(items0).isOk - check queue.push(items1).isOk - let last = items1[items1.high] - check eventually queue.contains(last) - queue.delete(last.requestId, last.slotIndex) - check not onProcessSlotCalledWith.anyIt(it == (last.requestId, last.slotIndex)) - - test "can delete all items by request id": - newSlotQueue(maxSize = 8, maxWorkers = 1, processSlotDelay = 10.millis) - let request0 = StorageRequest.example - var request1 = StorageRequest.example - request1.ask.collateralPerByte += 1.u256 - let items0 = - SlotQueueItem.init(request0, collateral = request0.ask.collateralPerSlot) - let items1 = - SlotQueueItem.init(request1, collateral = request1.ask.collateralPerSlot) - check queue.push(items0).isOk - check queue.push(items1).isOk - queue.delete(request1.id) - check not onProcessSlotCalledWith.anyIt(it[0] == request1.id) - - test "can check if contains item": - newSlotQueue(maxSize = 6, maxWorkers = 1, processSlotDelay = 10.millis) - let request0 = StorageRequest.example - var request1 = StorageRequest.example - var request2 = StorageRequest.example - var request3 = StorageRequest.example - var request4 = StorageRequest.example - var request5 = StorageRequest.example - request1.ask.collateralPerByte = request0.ask.collateralPerByte + 1 - request2.ask.collateralPerByte = request1.ask.collateralPerByte + 1 - request3.ask.collateralPerByte = request2.ask.collateralPerByte + 1 - request4.ask.collateralPerByte = request3.ask.collateralPerByte + 1 - request5.ask.collateralPerByte = request4.ask.collateralPerByte + 1 - let item0 = - SlotQueueItem.init(request0, 0, collateral = request0.ask.collateralPerSlot) - let item1 = - SlotQueueItem.init(request1, 0, collateral = request1.ask.collateralPerSlot) - let item2 = - SlotQueueItem.init(request2, 0, collateral = request2.ask.collateralPerSlot) - let item3 = - SlotQueueItem.init(request3, 0, collateral = request3.ask.collateralPerSlot) - let item4 = - SlotQueueItem.init(request4, 0, collateral = request4.ask.collateralPerSlot) - let item5 = - SlotQueueItem.init(request5, 0, collateral = request5.ask.collateralPerSlot) - check queue.contains(item5) == false - check queue.push(@[item0, item1, item2, item3, item4, item5]).isOk - check queue.contains(item5) - - test "sorts items by profitability descending (higher pricePerBytePerSecond == higher priority == goes first in the list)": - var request = StorageRequest.example - let item0 = - SlotQueueItem.init(request, 0, collateral = request.ask.collateralPerSlot) - request.ask.pricePerBytePerSecond += 1.u256 - let item1 = - SlotQueueItem.init(request, 1, collateral = request.ask.collateralPerSlot) - check item1 < item0 - - test "sorts items by collateral ascending (higher required collateral = lower priority == comes later in the list)": - var request = StorageRequest.example - let item0 = - SlotQueueItem.init(request, 0, collateral = request.ask.collateralPerSlot) - let item1 = SlotQueueItem.init( - request, 1, collateral = request.ask.collateralPerSlot + 1.u256 - ) - check item1 > item0 - - test "sorts items by expiry descending (longer expiry = higher priority)": - var request = StorageRequest.example - let item0 = SlotQueueItem.init( - request.id, 0, request.ask, expiry = 3, collateral = request.ask.collateralPerSlot - ) - let item1 = SlotQueueItem.init( - request.id, 1, request.ask, expiry = 7, collateral = request.ask.collateralPerSlot - ) - check item1 < item0 - - test "sorts items by slot size descending (bigger dataset = higher profitability = higher priority)": - var request = StorageRequest.example - let item0 = - SlotQueueItem.init(request, 0, collateral = request.ask.collateralPerSlot) - request.ask.slotSize += 1 - let item1 = - SlotQueueItem.init(request, 1, collateral = request.ask.collateralPerSlot) - check item1 < item0 - - test "should call callback once an item is added": - newSlotQueue(maxSize = 2, maxWorkers = 2) - let item = SlotQueueItem.example - check not onProcessSlotCalled - check queue.push(item).isOk - check eventually onProcessSlotCalled - - test "should only process item once": - newSlotQueue(maxSize = 2, maxWorkers = 2) - let item = SlotQueueItem.example - check queue.push(item).isOk - check eventually onProcessSlotCalledWith == @[(item.requestId, item.slotIndex)] - - test "processes items in order of addition when only one item is added at a time": - newSlotQueue(maxSize = 2, maxWorkers = 2) - # sleeping after push allows the slotqueue loop to iterate, - # calling the callback for each pushed/updated item - var request = StorageRequest.example - let item0 = - SlotQueueItem.init(request, 0, collateral = request.ask.collateralPerSlot) - request.ask.pricePerBytePerSecond += 1.u256 - let item1 = - SlotQueueItem.init(request, 1, collateral = request.ask.collateralPerSlot) - request.ask.pricePerBytePerSecond += 1.u256 - let item2 = - SlotQueueItem.init(request, 2, collateral = request.ask.collateralPerSlot) - request.ask.pricePerBytePerSecond += 1.u256 - let item3 = - SlotQueueItem.init(request, 3, collateral = request.ask.collateralPerSlot) - - check queue.push(item0).isOk - await sleepAsync(1.millis) - check queue.push(item1).isOk - await sleepAsync(1.millis) - check queue.push(item2).isOk - await sleepAsync(1.millis) - check queue.push(item3).isOk - - check eventually ( - onProcessSlotCalledWith == - @[ - (item0.requestId, item0.slotIndex), - (item1.requestId, item1.slotIndex), - (item2.requestId, item2.slotIndex), - (item3.requestId, item3.slotIndex), - ] - ) - - test "should process items in correct order according to the queue invariant when more than one item is added at a time": - newSlotQueue(maxSize = 4, maxWorkers = 2) - # sleeping after push allows the slotqueue loop to iterate, - # calling the callback for each pushed/updated item - var request = StorageRequest.example - let item0 = - SlotQueueItem.init(request, 0, collateral = request.ask.collateralPerSlot) - request.ask.pricePerBytePerSecond += 1.u256 - let item1 = - SlotQueueItem.init(request, 1, collateral = request.ask.collateralPerSlot) - request.ask.pricePerBytePerSecond += 1.u256 - let item2 = - SlotQueueItem.init(request, 2, collateral = request.ask.collateralPerSlot) - request.ask.pricePerBytePerSecond += 1.u256 - let item3 = - SlotQueueItem.init(request, 3, collateral = request.ask.collateralPerSlot) - - check queue.push(item0).isOk - check queue.push(item1).isOk - check queue.push(item2).isOk - check queue.push(item3).isOk - - await sleepAsync(1.millis) - - check eventually ( - onProcessSlotCalledWith == - @[ - (item3.requestId, item3.slotIndex), - (item2.requestId, item2.slotIndex), - (item1.requestId, item1.slotIndex), - (item0.requestId, item0.slotIndex), - ] - ) - - test "pushing items to queue unpauses queue": - newSlotQueue(maxSize = 4, maxWorkers = 4) - queue.pause - - let request = StorageRequest.example - var items = SlotQueueItem.init(request, collateral = request.ask.collateralPerSlot) - check queue.push(items).isOk - # check all items processed - check eventually queue.len == 0 - - test "pushing seen item does not unpause queue": - newSlotQueue(maxSize = 4, maxWorkers = 4) - let request = StorageRequest.example - let item0 = SlotQueueItem.init( - request.id, 0'u16, request.ask, 0, request.ask.collateralPerSlot, seen = true - ) - check queue.paused - check queue.push(item0).isOk - check queue.paused - - test "paused queue waits for unpause before continuing processing": - newSlotQueue(maxSize = 4, maxWorkers = 4) - let request = StorageRequest.example - let item = SlotQueueItem.init( - request.id, 1'u16, request.ask, 0, request.ask.collateralPerSlot, seen = false - ) - check queue.paused - # push causes unpause - check queue.push(item).isOk - # check all items processed - check eventually onProcessSlotCalledWith == @[(item.requestId, item.slotIndex)] - check eventually queue.len == 0 - - test "processing a 'seen' item pauses the queue": - newSlotQueue(maxSize = 4, maxWorkers = 4) - let request = StorageRequest.example - let unseen = SlotQueueItem.init( - request.id, 0'u16, request.ask, 0, request.ask.collateralPerSlot, seen = false - ) - let seen = SlotQueueItem.init( - request.id, 1'u16, request.ask, 0, request.ask.collateralPerSlot, seen = true - ) - # push causes unpause - check queue.push(unseen).isSuccess - # check all items processed - check eventually queue.len == 0 - # push seen item - check queue.push(seen).isSuccess - # queue should be paused - check eventually queue.paused - - test "item 'seen' flags can be cleared": - newSlotQueue(maxSize = 4, maxWorkers = 1) - let request = StorageRequest.example - let item0 = SlotQueueItem.init( - request.id, 0'u16, request.ask, 0, request.ask.collateralPerSlot, seen = true - ) - let item1 = SlotQueueItem.init( - request.id, 1'u16, request.ask, 0, request.ask.collateralPerSlot, seen = true - ) - check queue.push(item0).isOk - check queue.push(item1).isOk - check queue[0].seen - check queue[1].seen - - queue.clearSeenFlags() - check queue[0].seen == false - check queue[1].seen == false diff --git a/tests/codex/sales/teststates.nim b/tests/codex/sales/teststates.nim deleted file mode 100644 index fd918ccc..00000000 --- a/tests/codex/sales/teststates.nim +++ /dev/null @@ -1,16 +0,0 @@ -import ./states/testunknown -import ./states/testdownloading -import ./states/testfilling -import ./states/testpayout -import ./states/testfinished -import ./states/testinitialproving -import ./states/testfilled -import ./states/testproving -import ./states/testsimulatedproving -import ./states/testcancelled -import ./states/testerrored -import ./states/testignored -import ./states/testpreparing -import ./states/testslotreserving - -{.warning[UnusedImport]: off.} diff --git a/tests/codex/slots/backends/helpers.nim b/tests/codex/slots/backends/helpers.nim deleted file mode 100644 index e1b6822a..00000000 --- a/tests/codex/slots/backends/helpers.nim +++ /dev/null @@ -1,150 +0,0 @@ -import std/sequtils -import std/strutils -import std/options - -import pkg/poseidon2 -import pkg/poseidon2/io -import pkg/constantine/math/arithmetic -import pkg/constantine/math/io/io_bigints -import pkg/constantine/math/io/io_fields - -import pkg/codex/merkletree -import pkg/codex/slots -import pkg/codex/slots/types -import pkg/codex/utils/json - -export types - -func toJsonDecimal*(big: BigInt[254]): string = - let s = big.toDecimal.strip(leading = true, trailing = false, chars = {'0'}) - if s.len == 0: "0" else: s - -func toJson*(g1: CircomG1): JsonNode = - %*{ - "x": Bn254Fr.fromBytes(g1.x).get.toBig.toJsonDecimal, - "y": Bn254Fr.fromBytes(g1.y).get.toBig.toJsonDecimal, - } - -func toJson*(g2: CircomG2): JsonNode = - %*{ - "x": [ - Bn254Fr.fromBytes(g2.x[0]).get.toBig.toJsonDecimal, - Bn254Fr.fromBytes(g2.x[1]).get.toBig.toJsonDecimal, - ], - "y": [ - Bn254Fr.fromBytes(g2.y[0]).get.toBig.toJsonDecimal, - Bn254Fr.fromBytes(g2.y[1]).get.toBig.toJsonDecimal, - ], - } - -proc toJson*(vpk: VerifyingKey): JsonNode = - let ic = - toSeq(cast[ptr UncheckedArray[CircomG1]](vpk.ic).toOpenArray(0, vpk.icLen.int - 1)) - - echo ic.len - %*{ - "alpha1": vpk.alpha1.toJson, - "beta2": vpk.beta2.toJson, - "gamma2": vpk.gamma2.toJson, - "delta2": vpk.delta2.toJson, - "ic": ic.mapIt(it.toJson), - } - -func toJson*(input: ProofInputs[Poseidon2Hash]): JsonNode = - var input = input - - %*{ - "dataSetRoot": input.datasetRoot.toBig.toJsonDecimal, - "entropy": input.entropy.toBig.toJsonDecimal, - "nCellsPerSlot": input.nCellsPerSlot, - "nSlotsPerDataSet": input.nSlotsPerDataSet, - "slotIndex": input.slotIndex, - "slotRoot": input.slotRoot.toDecimal, - "slotProof": input.slotProof.mapIt(it.toBig.toJsonDecimal), - "cellData": input.samples.mapIt(it.cellData.mapIt(it.toBig.toJsonDecimal)), - "merklePaths": input.samples.mapIt(it.merklePaths.mapIt(it.toBig.toJsonDecimal)), - } - -func toJson*(input: NormalizedProofInputs[Poseidon2Hash]): JsonNode = - toJson(ProofInputs[Poseidon2Hash](input)) - -func jsonToProofInput*( - _: type Poseidon2Hash, inputJson: JsonNode -): ProofInputs[Poseidon2Hash] = - let - cellData = inputJson["cellData"].mapIt( - it.mapIt( - block: - var - big: BigInt[256] - hash: Poseidon2Hash - data: array[32, byte] - assert bool(big.fromDecimal(it.str)) - assert data.marshal(big, littleEndian) - - Poseidon2Hash.fromBytes(data).get - ).concat # flatten out elements - ) - - merklePaths = inputJson["merklePaths"].mapIt( - it.mapIt( - block: - var - big: BigInt[254] - hash: Poseidon2Hash - assert bool(big.fromDecimal(it.getStr)) - hash.fromBig(big) - hash - ) - ) - - slotProof = inputJson["slotProof"].mapIt( - block: - var - big: BigInt[254] - hash: Poseidon2Hash - assert bool(big.fromDecimal(it.str)) - hash.fromBig(big) - hash - ) - - datasetRoot = block: - var - big: BigInt[254] - hash: Poseidon2Hash - assert bool(big.fromDecimal(inputJson["dataSetRoot"].str)) - hash.fromBig(big) - hash - - slotRoot = block: - var - big: BigInt[254] - hash: Poseidon2Hash - assert bool(big.fromDecimal(inputJson["slotRoot"].str)) - hash.fromBig(big) - hash - - entropy = block: - var - big: BigInt[254] - hash: Poseidon2Hash - assert bool(big.fromDecimal(inputJson["entropy"].str)) - hash.fromBig(big) - hash - - nCellsPerSlot = inputJson["nCellsPerSlot"].getInt - nSlotsPerDataSet = inputJson["nSlotsPerDataSet"].getInt - slotIndex = inputJson["slotIndex"].getInt - - ProofInputs[Poseidon2Hash]( - entropy: entropy, - slotIndex: slotIndex, - datasetRoot: datasetRoot, - slotProof: slotProof, - slotRoot: slotRoot, - nCellsPerSlot: nCellsPerSlot, - nSlotsPerDataSet: nSlotsPerDataSet, - samples: zip(cellData, merklePaths).mapIt( - Sample[Poseidon2Hash](cellData: it[0], merklePaths: it[1]) - ), - ) diff --git a/tests/codex/slots/backends/testcircomcompat.nim b/tests/codex/slots/backends/testcircomcompat.nim deleted file mode 100644 index b61d4f18..00000000 --- a/tests/codex/slots/backends/testcircomcompat.nim +++ /dev/null @@ -1,115 +0,0 @@ -import std/options - -import ../../../asynctest - -import pkg/chronos -import pkg/poseidon2 -import pkg/serde/json - -import pkg/codex/slots {.all.} -import pkg/codex/slots/types {.all.} -import pkg/codex/merkletree -import pkg/codex/codextypes -import pkg/codex/manifest -import pkg/codex/stores - -import ./helpers -import ../helpers -import ../../helpers - -suite "Test Circom Compat Backend - control inputs": - let - r1cs = "tests/circuits/fixtures/proof_main.r1cs" - wasm = "tests/circuits/fixtures/proof_main.wasm" - zkey = "tests/circuits/fixtures/proof_main.zkey" - - var - circom: CircomCompat - proofInputs: ProofInputs[Poseidon2Hash] - - setup: - let - inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = !JsonNode.parse(inputData) - - proofInputs = Poseidon2Hash.jsonToProofInput(inputJson) - circom = CircomCompat.init(r1cs, wasm, zkey) - - teardown: - circom.release() # this comes from the rust FFI - - test "Should verify with correct inputs": - let proof = circom.prove(proofInputs).tryGet - - check circom.verify(proof, proofInputs).tryGet - - test "Should not verify with incorrect inputs": - proofInputs.slotIndex = 1 # change slot index - - let proof = circom.prove(proofInputs).tryGet - - check circom.verify(proof, proofInputs).tryGet == false - -suite "Test Circom Compat Backend": - let - ecK = 2 - ecM = 2 - slotId = 3 - samples = 5 - numDatasetBlocks = 8 - blockSize = DefaultBlockSize - cellSize = DefaultCellSize - - r1cs = "tests/circuits/fixtures/proof_main.r1cs" - wasm = "tests/circuits/fixtures/proof_main.wasm" - zkey = "tests/circuits/fixtures/proof_main.zkey" - - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - - var - store: BlockStore - manifest: Manifest - protected: Manifest - verifiable: Manifest - circom: CircomCompat - proofInputs: ProofInputs[Poseidon2Hash] - challenge: array[32, byte] - builder: Poseidon2Builder - sampler: Poseidon2Sampler - - setup: - let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() - - store = RepoStore.new(repoDs, metaDs) - - (manifest, protected, verifiable) = await createVerifiableManifest( - store, numDatasetBlocks, ecK, ecM, blockSize, cellSize - ) - - builder = Poseidon2Builder.new(store, verifiable).tryGet - sampler = Poseidon2Sampler.new(slotId, store, builder).tryGet - - circom = CircomCompat.init(r1cs, wasm, zkey) - challenge = 1234567.toF.toBytes.toArray32 - - proofInputs = (await sampler.getProofInput(challenge, samples)).tryGet - - teardown: - circom.release() # this comes from the rust FFI - await repoTmp.destroyDb() - await metaTmp.destroyDb() - - test "Should verify with correct input": - var proof = circom.prove(proofInputs).tryGet - - check circom.verify(proof, proofInputs).tryGet - - test "Should not verify with incorrect input": - proofInputs.slotIndex = 1 # change slot index - - let proof = circom.prove(proofInputs).tryGet - - check circom.verify(proof, proofInputs).tryGet == false diff --git a/tests/codex/slots/helpers.nim b/tests/codex/slots/helpers.nim index fced1f1c..d4facaca 100644 --- a/tests/codex/slots/helpers.nim +++ b/tests/codex/slots/helpers.nim @@ -10,7 +10,6 @@ import pkg/codex/manifest import pkg/codex/blocktype as bt import pkg/codex/chunker import pkg/codex/indexingstrategy -import pkg/codex/slots import pkg/codex/rng import ../helpers @@ -84,89 +83,3 @@ proc createBlocks*( let blk = bt.Block.new(chunk).tryGet() discard await store.putBlock(blk) blk - -proc createProtectedManifest*( - datasetBlocks: seq[bt.Block], - store: BlockStore, - numDatasetBlocks: int, - ecK: int, - ecM: int, - blockSize: NBytes, - originalDatasetSize: int, - totalDatasetSize: int, -): Future[tuple[manifest: Manifest, protected: Manifest]] {.async.} = - let - cids = datasetBlocks.mapIt(it.cid) - datasetTree = CodexTree.init(cids[0 ..< numDatasetBlocks]).tryGet() - datasetTreeCid = datasetTree.rootCid().tryGet() - - protectedTree = CodexTree.init(cids).tryGet() - protectedTreeCid = protectedTree.rootCid().tryGet() - - for index, cid in cids[0 ..< numDatasetBlocks]: - let proof = datasetTree.getProof(index).tryGet() - (await store.putCidAndProof(datasetTreeCid, index, cid, proof)).tryGet - - for index, cid in cids: - let proof = protectedTree.getProof(index).tryGet() - (await store.putCidAndProof(protectedTreeCid, index, cid, proof)).tryGet - - let - manifest = Manifest.new( - treeCid = datasetTreeCid, - blockSize = blockSize, - datasetSize = originalDatasetSize.NBytes, - ) - - protectedManifest = Manifest.new( - manifest = manifest, - treeCid = protectedTreeCid, - datasetSize = totalDatasetSize.NBytes, - ecK = ecK, - ecM = ecM, - strategy = SteppedStrategy, - ) - - manifestBlock = - bt.Block.new(manifest.encode().tryGet(), codec = ManifestCodec).tryGet() - - protectedManifestBlock = - bt.Block.new(protectedManifest.encode().tryGet(), codec = ManifestCodec).tryGet() - - (await store.putBlock(manifestBlock)).tryGet() - (await store.putBlock(protectedManifestBlock)).tryGet() - - (manifest, protectedManifest) - -proc createVerifiableManifest*( - store: BlockStore, - numDatasetBlocks: int, - ecK: int, - ecM: int, - blockSize: NBytes, - cellSize: NBytes, -): Future[tuple[manifest: Manifest, protected: Manifest, verifiable: Manifest]] {. - async -.} = - let - numSlots = ecK + ecM - numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) - # total number of blocks in the dataset after - # EC (should will match number of slots) - originalDatasetSize = numDatasetBlocks * blockSize.int - totalDatasetSize = numTotalBlocks * blockSize.int - - chunker = - RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize) - datasetBlocks = await chunker.createBlocks(store) - - (manifest, protectedManifest) = await createProtectedManifest( - datasetBlocks, store, numDatasetBlocks, ecK, ecM, blockSize, originalDatasetSize, - totalDatasetSize, - ) - - builder = Poseidon2Builder.new(store, protectedManifest, cellSize = cellSize).tryGet - verifiableManifest = (await builder.buildManifest()).tryGet - - # build the slots and manifest - (manifest, protectedManifest, verifiableManifest) diff --git a/tests/codex/slots/sampler/testsampler.nim b/tests/codex/slots/sampler/testsampler.nim deleted file mode 100644 index 78b245a3..00000000 --- a/tests/codex/slots/sampler/testsampler.nim +++ /dev/null @@ -1,167 +0,0 @@ -import std/sequtils -import std/options - -import ../../../asynctest - -import pkg/questionable/results - -import pkg/codex/stores -import pkg/codex/merkletree -import pkg/codex/utils/json -import pkg/codex/codextypes -import pkg/codex/slots -import pkg/codex/slots/builder -import pkg/codex/utils/poseidon2digest -import pkg/codex/slots/sampler/utils - -import pkg/constantine/math/arithmetic -import pkg/constantine/math/io/io_bigints - -import ../backends/helpers -import ../helpers -import ../../helpers - -suite "Test Sampler - control samples": - var - inputData: string - inputJson: JsonNode - proofInput: ProofInputs[Poseidon2Hash] - - setup: - inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = !JsonNode.parse(inputData) - proofInput = Poseidon2Hash.jsonToProofInput(inputJson) - - test "Should verify control samples": - let - blockCells = 32 - cellIdxs = - proofInput.entropy.cellIndices(proofInput.slotRoot, proofInput.nCellsPerSlot, 5) - - for i, cellIdx in cellIdxs: - let - sample = proofInput.samples[i] - cellIdx = cellIdxs[i] - - cellProof = Poseidon2Proof.init( - cellIdx.toCellInBlk(blockCells), - proofInput.nCellsPerSlot, - sample.merklePaths[0 ..< 5], - ).tryGet - - slotProof = Poseidon2Proof.init( - cellIdx.toBlkInSlot(blockCells), - proofInput.nCellsPerSlot, - sample.merklePaths[5 ..< 9], - ).tryGet - - cellData = sample.cellData - cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet - slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet - - check slotProof.verify(slotLeaf, proofInput.slotRoot).tryGet - - test "Should verify control dataset root": - let datasetProof = Poseidon2Proof.init( - proofInput.slotIndex, proofInput.nSlotsPerDataSet, proofInput.slotProof[0 ..< 4] - ).tryGet - - check datasetProof.verify(proofInput.slotRoot, proofInput.datasetRoot).tryGet - -suite "Test Sampler": - let - slotIndex = 3 - nSamples = 5 - ecK = 3 - ecM = 2 - datasetBlocks = 8 - entropy = 1234567.toF - blockSize = DefaultBlockSize - cellSize = DefaultCellSize - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - - var - store: RepoStore - builder: Poseidon2Builder - manifest: Manifest - protected: Manifest - verifiable: Manifest - - setup: - let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() - - store = RepoStore.new(repoDs, metaDs) - - (manifest, protected, verifiable) = await createVerifiableManifest( - store, datasetBlocks, ecK, ecM, blockSize, cellSize - ) - - # create sampler - builder = Poseidon2Builder.new(store, verifiable).tryGet - - teardown: - await store.close() - await repoTmp.destroyDb() - await metaTmp.destroyDb() - - test "Should fail instantiating for invalid slot index": - let sampler = Poseidon2Sampler.new(builder.slotRoots.len, store, builder) - - check sampler.isErr - - test "Should fail instantiating for non verifiable builder": - let - nonVerifiableBuilder = Poseidon2Builder.new(store, protected).tryGet - sampler = Poseidon2Sampler.new(slotIndex, store, nonVerifiableBuilder) - - check sampler.isErr - - test "Should verify samples": - let - sampler = Poseidon2Sampler.new(slotIndex, store, builder).tryGet - - verifyTree = builder.verifyTree.get # get the dataset tree - slotProof = verifyTree.getProof(slotIndex).tryGet # get slot proof for index - datasetRoot = verifyTree.root().tryGet # get dataset root - slotTreeCid = verifiable.slotRoots[slotIndex] - # get slot tree cid to retrieve proof from storage - slotRoot = builder.slotRoots[slotIndex] # get slot root hash - cellIdxs = entropy.cellIndices(slotRoot, builder.numSlotCells, nSamples) - - nBlockCells = builder.numBlockCells - nSlotCells = builder.numSlotCells - - for i, cellIdx in cellIdxs: - let - sample = (await sampler.getSample(cellIdx, slotTreeCid, slotRoot)).tryGet - - cellProof = Poseidon2Proof.init( - cellIdx.toCellInBlk(nBlockCells), nSlotCells, sample.merklePaths[0 ..< 5] - ).tryGet - - slotProof = Poseidon2Proof.init( - cellIdx.toBlkInSlot(nBlockCells), - nSlotCells, - sample.merklePaths[5 ..< sample.merklePaths.len], - ).tryGet - - cellData = sample.cellData - cellLeaf = Poseidon2Hash.spongeDigest(cellData, rate = 2).tryGet - slotLeaf = cellProof.reconstructRoot(cellLeaf).tryGet - - check slotProof.verify(slotLeaf, slotRoot).tryGet - - test "Should verify dataset root": - let - sampler = Poseidon2Sampler.new(slotIndex, store, builder).tryGet - proofInput = - (await sampler.getProofInput(entropy.toBytes.toArray32, nSamples)).tryGet - - datasetProof = Poseidon2Proof.init( - proofInput.slotIndex, builder.slotRoots.len, proofInput.slotProof - ).tryGet - - check datasetProof.verify(builder.slotRoots[slotIndex], builder.verifyRoot.get).tryGet diff --git a/tests/codex/slots/sampler/testutils.nim b/tests/codex/slots/sampler/testutils.nim deleted file mode 100644 index f20b5efc..00000000 --- a/tests/codex/slots/sampler/testutils.nim +++ /dev/null @@ -1,107 +0,0 @@ -import std/sugar - -import ../../../asynctest - -import pkg/questionable/results -import pkg/constantine/math/arithmetic -import pkg/constantine/math/io/io_fields -import pkg/poseidon2/io -import pkg/poseidon2 -import pkg/chronos -import pkg/codex/chunker -import pkg/codex/stores -import pkg/codex/blocktype as bt -import pkg/codex/contracts/requests -import pkg/codex/contracts -import pkg/codex/merkletree -import pkg/codex/stores/cachestore -import pkg/codex/slots/types -import pkg/codex/slots/sampler/utils -import pkg/codex/utils/json - -import ../backends/helpers -import ../../helpers -import ../../examples -import ../../merkletree/helpers - -asyncchecksuite "Test proof sampler utils": - let cellsPerBlock = DefaultBlockSize div DefaultCellSize - - var - inputData: string - inputJson: JsonNode - proofInput: ProofInputs[Poseidon2Hash] - - setup: - inputData = readFile("tests/circuits/fixtures/input.json") - inputJson = !JsonNode.parse(inputData) - proofInput = Poseidon2Hash.jsonToProofInput(inputJson) - - test "Extract low bits": - proc extract(value: uint64, nBits: int): uint64 = - let big = toF(value).toBig() - return extractLowBits(big, nBits) - - check: - extract(0x88, 4) == 0x8.uint64 - extract(0x88, 7) == 0x8.uint64 - extract(0x9A, 5) == 0x1A.uint64 - extract(0x9A, 7) == 0x1A.uint64 - extract(0x1248, 10) == 0x248.uint64 - extract(0x1248, 12) == 0x248.uint64 - extract(0x1248306A560C9AC0.uint64, 10) == 0x2C0.uint64 - extract(0x1248306A560C9AC0.uint64, 12) == 0xAC0.uint64 - extract(0x1248306A560C9AC0.uint64, 50) == 0x306A560C9AC0.uint64 - extract(0x1248306A560C9AC0.uint64, 52) == 0x8306A560C9AC0.uint64 - - test "Can find single slot-cell index": - proc slotCellIndex(i: Natural): Natural = - return - cellIndex(proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i) - - proc getExpectedIndex(i: int): Natural = - let hash = - Sponge.digest(@[proofInput.entropy, proofInput.slotRoot, toF(i)], rate = 2) - - return int(extractLowBits(hash.toBig(), ceilingLog2(proofInput.nCellsPerSlot))) - - check: - slotCellIndex(1) == getExpectedIndex(1) - slotCellIndex(2) == getExpectedIndex(2) - slotCellIndex(3) == getExpectedIndex(3) - - test "Can find sequence of slot-cell indices": - proc slotCellIndices(n: int): seq[Natural] = - cellIndices( - proofInput.entropy, proofInput.slotRoot, numCells = proofInput.nCellsPerSlot, n - ) - - proc getExpectedIndices(n: int): seq[Natural] = - return collect( - newSeq, - (; - for i in 1 .. n: - cellIndex( - proofInput.entropy, proofInput.slotRoot, proofInput.nCellsPerSlot, i - ) - ), - ) - - check: - slotCellIndices(3) == getExpectedIndices(3) - - for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]: - test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & - ")": - let slotBlockIndex = toBlkInSlot(input, numCells = cellsPerBlock) - - check: - slotBlockIndex == expected - - for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: - test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & - ")": - let blockCellIndex = toCellInBlk(input, numCells = cellsPerBlock) - - check: - blockCellIndex == expected diff --git a/tests/codex/slots/testbackendfactory.nim b/tests/codex/slots/testbackendfactory.nim deleted file mode 100644 index a24bc41a..00000000 --- a/tests/codex/slots/testbackendfactory.nim +++ /dev/null @@ -1,97 +0,0 @@ -import os -import ../../asynctest - -import pkg/chronos -import pkg/confutils/defs -import pkg/codex/conf -import pkg/codex/slots/proofs/backends -import pkg/codex/slots/proofs/backendfactory -import pkg/codex/slots/proofs/backendutils -import pkg/codex/utils/natutils - -import ../helpers -import ../examples - -type BackendUtilsMock = ref object of BackendUtils - argR1csFile: string - argWasmFile: string - argZKeyFile: string - -method initializeCircomBackend*( - self: BackendUtilsMock, r1csFile: string, wasmFile: string, zKeyFile: string -): AnyBackend = - self.argR1csFile = r1csFile - self.argWasmFile = wasmFile - self.argZKeyFile = zKeyFile - # We return a backend with *something* that's not nil that we can check for. - var - key = VerifyingKey(icLen: 123) - vkpPtr: ptr VerifyingKey = key.addr - return CircomCompat(vkp: vkpPtr) - -suite "Test BackendFactory": - let - utilsMock = BackendUtilsMock() - circuitDir = "testecircuitdir" - - setup: - createDir(circuitDir) - - teardown: - removeDir(circuitDir) - - test "Should create backend from cli config": - let - config = CodexConf( - cmd: StartUpCmd.persistence, - nat: NatConfig(hasExtIp: false, nat: NatNone), - metricsAddress: parseIpAddress("127.0.0.1"), - persistenceCmd: PersistenceCmd.prover, - marketplaceAddress: EthAddress.example.some, - circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), - circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), - circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"), - ) - backend = config.initializeBackend(utilsMock).tryGet - - check: - backend.vkp != nil - utilsMock.argR1csFile == $config.circomR1cs - utilsMock.argWasmFile == $config.circomWasm - utilsMock.argZKeyFile == $config.circomZkey - - test "Should create backend from local files": - let - config = CodexConf( - cmd: StartUpCmd.persistence, - nat: NatConfig(hasExtIp: false, nat: NatNone), - metricsAddress: parseIpAddress("127.0.0.1"), - persistenceCmd: PersistenceCmd.prover, - marketplaceAddress: EthAddress.example.some, - - # Set the circuitDir such that the tests/circuits/fixtures/ files - # will be picked up as local files: - circuitDir: OutDir("tests/circuits/fixtures"), - ) - backend = config.initializeBackend(utilsMock).tryGet - - check: - backend.vkp != nil - utilsMock.argR1csFile == config.circuitDir / "proof_main.r1cs" - utilsMock.argWasmFile == config.circuitDir / "proof_main.wasm" - utilsMock.argZKeyFile == config.circuitDir / "proof_main.zkey" - - test "Should suggest usage of downloader tool when files not available": - let - config = CodexConf( - cmd: StartUpCmd.persistence, - nat: NatConfig(hasExtIp: false, nat: NatNone), - metricsAddress: parseIpAddress("127.0.0.1"), - persistenceCmd: PersistenceCmd.prover, - marketplaceAddress: EthAddress.example.some, - circuitDir: OutDir(circuitDir), - ) - backendResult = config.initializeBackend(utilsMock) - - check: - backendResult.isErr diff --git a/tests/codex/slots/testbackends.nim b/tests/codex/slots/testbackends.nim deleted file mode 100644 index b9994fcd..00000000 --- a/tests/codex/slots/testbackends.nim +++ /dev/null @@ -1,3 +0,0 @@ -import ./backends/testcircomcompat - -{.warning[UnusedImport]: off.} diff --git a/tests/codex/slots/testconverters.nim b/tests/codex/slots/testconverters.nim deleted file mode 100644 index 58857f6b..00000000 --- a/tests/codex/slots/testconverters.nim +++ /dev/null @@ -1,46 +0,0 @@ -import pkg/chronos -import pkg/poseidon2 -import pkg/poseidon2/io -import pkg/constantine/math/io/io_fields -import pkg/questionable/results -import pkg/codex/merkletree -import pkg/codex/slots/converters - -import ../../asynctest -import ../examples -import ../merkletree/helpers - -let hash: Poseidon2Hash = toF(12345) - -suite "Converters": - test "CellBlock cid": - let - cid = toCellCid(hash).tryGet() - value = fromCellCid(cid).tryGet() - - check: - hash.toDecimal() == value.toDecimal() - - test "Slot cid": - let - cid = toSlotCid(hash).tryGet() - value = fromSlotCid(cid).tryGet() - - check: - hash.toDecimal() == value.toDecimal() - - test "Verify cid": - let - cid = toVerifyCid(hash).tryGet() - value = fromVerifyCid(cid).tryGet() - - check: - hash.toDecimal() == value.toDecimal() - - test "Proof": - let - codexProof = toEncodableProof(Poseidon2Proof.example).tryGet() - poseidonProof = toVerifiableProof(codexProof).tryGet() - - check: - Poseidon2Proof.example == poseidonProof diff --git a/tests/codex/slots/testprover.nim b/tests/codex/slots/testprover.nim deleted file mode 100644 index c567db55..00000000 --- a/tests/codex/slots/testprover.nim +++ /dev/null @@ -1,88 +0,0 @@ -import ../../asynctest - -import pkg/chronos -import pkg/libp2p/cid - -import pkg/codex/merkletree -import pkg/codex/chunker -import pkg/codex/blocktype as bt -import pkg/codex/slots -import pkg/codex/stores -import pkg/codex/conf -import pkg/confutils/defs -import pkg/poseidon2/io -import pkg/codex/utils/poseidon2digest -import pkg/codex/nat -import pkg/codex/utils/natutils -import ./helpers -import ../helpers - -suite "Test Prover": - let - samples = 5 - blockSize = DefaultBlockSize - cellSize = DefaultCellSize - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - challenge = 1234567.toF.toBytes.toArray32 - - var - store: BlockStore - prover: Prover - - setup: - let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() - config = CodexConf( - cmd: StartUpCmd.persistence, - nat: NatConfig(hasExtIp: false, nat: NatNone), - metricsAddress: parseIpAddress("127.0.0.1"), - persistenceCmd: PersistenceCmd.prover, - circomR1cs: InputFile("tests/circuits/fixtures/proof_main.r1cs"), - circomWasm: InputFile("tests/circuits/fixtures/proof_main.wasm"), - circomZkey: InputFile("tests/circuits/fixtures/proof_main.zkey"), - numProofSamples: samples, - ) - backend = config.initializeBackend().tryGet() - - store = RepoStore.new(repoDs, metaDs) - prover = Prover.new(store, backend, config.numProofSamples) - - teardown: - await repoTmp.destroyDb() - await metaTmp.destroyDb() - - test "Should sample and prove a slot": - let (_, _, verifiable) = await createVerifiableManifest( - store, - 8, # number of blocks in the original dataset (before EC) - 5, # ecK - 3, # ecM - blockSize, - cellSize, - ) - - let (inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet - - check: - (await prover.verify(proof, inputs)).tryGet == true - - test "Should generate valid proofs when slots consist of single blocks": - # To get single-block slots, we just need to set the number of blocks in - # the original dataset to be the same as ecK. The total number of blocks - # after generating random data for parity will be ecK + ecM, which will - # match the number of slots. - let (_, _, verifiable) = await createVerifiableManifest( - store, - 2, # number of blocks in the original dataset (before EC) - 2, # ecK - 1, # ecM - blockSize, - cellSize, - ) - - let (inputs, proof) = (await prover.prove(1, verifiable, challenge)).tryGet - - check: - (await prover.verify(proof, inputs)).tryGet == true diff --git a/tests/codex/slots/testsampler.nim b/tests/codex/slots/testsampler.nim deleted file mode 100644 index 50a40c2c..00000000 --- a/tests/codex/slots/testsampler.nim +++ /dev/null @@ -1,4 +0,0 @@ -import ./sampler/testsampler -import ./sampler/testutils - -{.warning[UnusedImport]: off.} diff --git a/tests/codex/slots/testslotbuilder.nim b/tests/codex/slots/testslotbuilder.nim deleted file mode 100644 index fc3c7bd5..00000000 --- a/tests/codex/slots/testslotbuilder.nim +++ /dev/null @@ -1,341 +0,0 @@ -import std/math -import std/importutils - -import ../../asynctest - -import pkg/chronos -import pkg/questionable/results -import pkg/codex/blocktype as bt -import pkg/codex/rng -import pkg/codex/stores -import pkg/codex/chunker -import pkg/codex/merkletree -import pkg/codex/manifest {.all.} -import pkg/codex/utils -import pkg/codex/utils/digest -import pkg/poseidon2 -import pkg/poseidon2/io - -import ./helpers -import ../helpers -import ../examples -import ../merkletree/helpers - -import pkg/codex/indexingstrategy {.all.} -import pkg/codex/slots {.all.} - -privateAccess(Poseidon2Builder) # enable access to private fields -privateAccess(Manifest) # enable access to private fields - -const Strategy = LinearStrategy - -suite "Slot builder": - let - blockSize = NBytes 1024 - cellSize = NBytes 64 - ecK = 3 - ecM = 2 - - numSlots = ecK + ecM - numDatasetBlocks = 8 - numTotalBlocks = calcEcBlocksCount(numDatasetBlocks, ecK, ecM) - # total number of blocks in the dataset after - # EC (should will match number of slots) - originalDatasetSize = numDatasetBlocks * blockSize.int - totalDatasetSize = numTotalBlocks * blockSize.int - - numSlotBlocks = numTotalBlocks div numSlots - numBlockCells = (blockSize div cellSize).int # number of cells per block - numSlotCells = numSlotBlocks * numBlockCells # number of uncorrected slot cells - pow2SlotCells = nextPowerOfTwo(numSlotCells) # pow2 cells per slot - numPadSlotBlocks = (pow2SlotCells div numBlockCells) - numSlotBlocks - # pow2 blocks per slot - - numSlotBlocksTotal = - # pad blocks per slot - if numPadSlotBlocks > 0: - numPadSlotBlocks + numSlotBlocks - else: - numSlotBlocks - - numBlocksTotal = numSlotBlocksTotal * numSlots - - # empty digest - emptyDigest = SpongeMerkle.digest(newSeq[byte](blockSize.int), cellSize.int) - repoTmp = TempLevelDb.new() - metaTmp = TempLevelDb.new() - - var - datasetBlocks: seq[bt.Block] - localStore: BlockStore - manifest: Manifest - protectedManifest: Manifest - builder: Poseidon2Builder - chunker: Chunker - - setup: - let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() - - localStore = RepoStore.new(repoDs, metaDs) - chunker = - RandomChunker.new(Rng.instance(), size = totalDatasetSize, chunkSize = blockSize) - datasetBlocks = await chunker.createBlocks(localStore) - - (manifest, protectedManifest) = await createProtectedManifest( - datasetBlocks, localStore, numDatasetBlocks, ecK, ecM, blockSize, - originalDatasetSize, totalDatasetSize, - ) - - teardown: - await localStore.close() - await repoTmp.destroyDb() - await metaTmp.destroyDb() - - # TODO: THIS IS A BUG IN asynctest, because it doesn't release the - # objects after the test is done, so we need to do it manually - # - # Need to reset all objects because otherwise they get - # captured by the test runner closures, not good! - reset(datasetBlocks) - reset(localStore) - reset(manifest) - reset(protectedManifest) - reset(builder) - reset(chunker) - - test "Can only create builder with protected manifest": - let unprotectedManifest = Manifest.new( - treeCid = Cid.example, - blockSize = blockSize.NBytes, - datasetSize = originalDatasetSize.NBytes, - ) - - check: - Poseidon2Builder.new(localStore, unprotectedManifest, cellSize = cellSize).error.msg == - "Manifest is not protected." - - test "Number of blocks must be devisable by number of slots": - let mismatchManifest = Manifest.new( - manifest = Manifest.new( - treeCid = Cid.example, - blockSize = blockSize.NBytes, - datasetSize = originalDatasetSize.NBytes, - ), - treeCid = Cid.example, - datasetSize = totalDatasetSize.NBytes, - ecK = ecK - 1, - ecM = ecM, - strategy = Strategy, - ) - - check: - Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize).error.msg == - "Number of blocks must be divisible by number of slots." - - test "Block size must be divisable by cell size": - let mismatchManifest = Manifest.new( - manifest = Manifest.new( - treeCid = Cid.example, - blockSize = (blockSize + 1).NBytes, - datasetSize = (originalDatasetSize - 1).NBytes, - ), - treeCid = Cid.example, - datasetSize = (totalDatasetSize - 1).NBytes, - ecK = ecK, - ecM = ecM, - strategy = Strategy, - ) - - check: - Poseidon2Builder.new(localStore, mismatchManifest, cellSize = cellSize).error.msg == - "Block size must be divisible by cell size." - - test "Should build correct slot builder": - builder = - Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet() - - check: - builder.cellSize == cellSize - builder.numSlots == numSlots - builder.numBlockCells == numBlockCells - builder.numSlotBlocks == numSlotBlocksTotal - builder.numSlotCells == pow2SlotCells - builder.numBlocks == numBlocksTotal - - test "Should build slot hashes for all slots": - let - linearStrategy = Strategy.init( - 0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks - ) - - builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) - .tryGet() - - for i in 0 ..< numSlots: - let - expectedHashes = collect(newSeq): - for j, idx in linearStrategy.getIndices(i): - if j > (protectedManifest.numSlotBlocks - 1): - emptyDigest - else: - SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - - cellHashes = (await builder.getCellHashes(i)).tryGet() - - check: - cellHashes.len == expectedHashes.len - cellHashes == expectedHashes - - test "Should build slot trees for all slots": - let - linearStrategy = Strategy.init( - 0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks - ) - - builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) - .tryGet() - - for i in 0 ..< numSlots: - let - expectedHashes = collect(newSeq): - for j, idx in linearStrategy.getIndices(i): - if j > (protectedManifest.numSlotBlocks - 1): - emptyDigest - else: - SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - - expectedRoot = Merkle.digest(expectedHashes) - slotTree = (await builder.buildSlotTree(i)).tryGet() - - check: - slotTree.root().tryGet() == expectedRoot - - test "Should persist trees for all slots": - let builder = - Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet() - - for i in 0 ..< numSlots: - let - slotTree = (await builder.buildSlotTree(i)).tryGet() - slotRoot = (await builder.buildSlot(i)).tryGet() - slotCid = slotRoot.toSlotCid().tryGet() - - for cellIndex in 0 ..< numPadSlotBlocks: - let - (cellCid, proof) = - (await localStore.getCidAndProof(slotCid, cellIndex)).tryGet() - verifiableProof = proof.toVerifiableProof().tryGet() - posProof = slotTree.getProof(cellIndex).tryGet() - - check: - verifiableProof.path == posProof.path - verifiableProof.index == posProof.index - verifiableProof.nleaves == posProof.nleaves - - test "Should build correct verification root": - let - linearStrategy = Strategy.init( - 0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks - ) - builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) - .tryGet() - - (await builder.buildSlots()).tryGet - let - slotsHashes = collect(newSeq): - for i in 0 ..< numSlots: - let slotHashes = collect(newSeq): - for j, idx in linearStrategy.getIndices(i): - if j > (protectedManifest.numSlotBlocks - 1): - emptyDigest - else: - SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - - Merkle.digest(slotHashes) - - expectedRoot = Merkle.digest(slotsHashes) - rootHash = builder.buildVerifyTree(builder.slotRoots).tryGet().root.tryGet() - - check: - expectedRoot == rootHash - - test "Should build correct verification root manifest": - let - linearStrategy = Strategy.init( - 0, protectedManifest.blocksCount - 1, numSlots, numSlots, numPadSlotBlocks - ) - builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) - .tryGet() - - slotsHashes = collect(newSeq): - for i in 0 ..< numSlots: - let slotHashes = collect(newSeq): - for j, idx in linearStrategy.getIndices(i): - if j > (protectedManifest.numSlotBlocks - 1): - emptyDigest - else: - SpongeMerkle.digest(datasetBlocks[idx].data, cellSize.int) - - Merkle.digest(slotHashes) - - expectedRoot = Merkle.digest(slotsHashes) - manifest = (await builder.buildManifest()).tryGet() - mhash = manifest.verifyRoot.mhash.tryGet() - mhashBytes = mhash.digestBytes - rootHash = Poseidon2Hash.fromBytes(mhashBytes.toArray32).get - - check: - expectedRoot == rootHash - - test "Should not build from verifiable manifest with 0 slots": - var - builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) - .tryGet() - verifyManifest = (await builder.buildManifest()).tryGet() - - verifyManifest.slotRoots = @[] - check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr - - test "Should not build from verifiable manifest with incorrect number of slots": - var - builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) - .tryGet() - - verifyManifest = (await builder.buildManifest()).tryGet() - - verifyManifest.slotRoots.del(verifyManifest.slotRoots.len - 1) - - check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr - - test "Should not build from verifiable manifest with invalid verify root": - let builder = - Poseidon2Builder.new(localStore, protectedManifest, cellSize = cellSize).tryGet() - - var verifyManifest = (await builder.buildManifest()).tryGet() - - rng.shuffle(Rng.instance, verifyManifest.verifyRoot.data.buffer) - - check Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).isErr - - test "Should build from verifiable manifest": - let - builder = Poseidon2Builder - .new(localStore, protectedManifest, cellSize = cellSize) - .tryGet() - - verifyManifest = (await builder.buildManifest()).tryGet() - - verificationBuilder = - Poseidon2Builder.new(localStore, verifyManifest, cellSize = cellSize).tryGet() - - check: - builder.slotRoots == verificationBuilder.slotRoots - builder.verifyRoot == verificationBuilder.verifyRoot diff --git a/tests/codex/testerasure.nim b/tests/codex/testerasure.nim deleted file mode 100644 index 5046bac2..00000000 --- a/tests/codex/testerasure.nim +++ /dev/null @@ -1,374 +0,0 @@ -import std/sequtils -import std/sugar -import std/times - -import pkg/chronos -import pkg/questionable/results - -import pkg/codex/erasure -import pkg/codex/manifest -import pkg/codex/stores -import pkg/codex/blocktype as bt -import pkg/codex/rng -import pkg/codex/utils -import pkg/codex/indexingstrategy -import pkg/taskpools -import pkg/codex/utils/arrayutils - -import ../asynctest -import ./helpers -import ./examples - -suite "Erasure encode/decode": - const BlockSize = 1024'nb - const dataSetSize = BlockSize * 123 # weird geometry - - var rng: Rng - var chunker: Chunker - var manifest: Manifest - var store: BlockStore - var erasure: Erasure - let repoTmp = TempLevelDb.new() - let metaTmp = TempLevelDb.new() - var taskpool: Taskpool - - setup: - let - repoDs = repoTmp.newDb() - metaDs = metaTmp.newDb() - rng = Rng.instance() - chunker = RandomChunker.new(rng, size = dataSetSize, chunkSize = BlockSize) - store = RepoStore.new(repoDs, metaDs) - taskpool = Taskpool.new() - erasure = Erasure.new(store, leoEncoderProvider, leoDecoderProvider, taskpool) - manifest = await storeDataGetManifest(store, chunker) - - teardown: - await repoTmp.destroyDb() - await metaTmp.destroyDb() - taskpool.shutdown() - - proc encode(buffers, parity: int): Future[Manifest] {.async.} = - let encoded = - (await erasure.encode(manifest, buffers.Natural, parity.Natural)).tryGet() - - check: - encoded.blocksCount mod (buffers + parity) == 0 - encoded.rounded == roundUp(manifest.blocksCount, buffers) - encoded.steps == encoded.rounded div buffers - - return encoded - - test "Should tolerate losing M data blocks in a single random column": - const - buffers = 20 - parity = 10 - - let encoded = await encode(buffers, parity) - - var - column = rng.rand((encoded.blocksCount div encoded.steps) - 1) # random column - dropped: seq[int] - - for _ in 0 ..< encoded.ecM: - dropped.add(column) - (await store.delBlock(encoded.treeCid, column)).tryGet() - (await store.delBlock(manifest.treeCid, column)).tryGet() - column = (column + encoded.steps) mod encoded.blocksCount # wrap around - - var decoded = (await erasure.decode(encoded)).tryGet() - - check: - decoded.treeCid == manifest.treeCid - decoded.treeCid == encoded.originalTreeCid - decoded.blocksCount == encoded.originalBlocksCount - - for d in dropped: - if d < manifest.blocksCount: # we don't support returning parity blocks yet - let present = await store.hasBlock(manifest.treeCid, d) - check present.tryGet() - - test "Should not tolerate losing more than M data blocks in a single random column": - const - buffers = 20 - parity = 10 - - let encoded = await encode(buffers, parity) - - var - column = rng.rand((encoded.blocksCount div encoded.steps) - 1) # random column - dropped: seq[int] - - for _ in 0 ..< encoded.ecM + 1: - dropped.add(column) - (await store.delBlock(encoded.treeCid, column)).tryGet() - (await store.delBlock(manifest.treeCid, column)).tryGet() - column = (column + encoded.steps) mod encoded.blocksCount # wrap around - - var decoded: Manifest - - expect ResultFailure: - decoded = (await erasure.decode(encoded)).tryGet() - - for d in dropped: - let present = await store.hasBlock(manifest.treeCid, d) - check not present.tryGet() - - test "Should tolerate losing M data blocks in M random columns": - const - buffers = 20 - parity = 10 - - let encoded = await encode(buffers, parity) - - var - blocks: seq[int] - offset = 0 - - while offset < encoded.steps - 1: - let blockIdx = toSeq(countup(offset, encoded.blocksCount - 1, encoded.steps)) - - for _ in 0 ..< encoded.ecM: - blocks.add(rng.sample(blockIdx, blocks)) - offset.inc - - for idx in blocks: - (await store.delBlock(encoded.treeCid, idx)).tryGet() - (await store.delBlock(manifest.treeCid, idx)).tryGet() - discard - - discard (await erasure.decode(encoded)).tryGet() - - for d in 0 ..< manifest.blocksCount: - let present = await store.hasBlock(manifest.treeCid, d) - check present.tryGet() - - test "Should not tolerate losing more than M data blocks in M random columns": - const - buffers = 20 - parity = 10 - - let encoded = await encode(buffers, parity) - - var - blocks: seq[int] - offset = 0 - - while offset < encoded.steps: - let blockIdx = toSeq(countup(offset, encoded.blocksCount - 1, encoded.steps)) - - for _ in 0 ..< encoded.ecM + 1: # NOTE: the +1 - var idx: int - while true: - idx = rng.sample(blockIdx, blocks) - let blk = (await store.getBlock(encoded.treeCid, idx)).tryGet() - if not blk.isEmpty: - break - - blocks.add(idx) - offset.inc - - for idx in blocks: - (await store.delBlock(encoded.treeCid, idx)).tryGet() - (await store.delBlock(manifest.treeCid, idx)).tryGet() - discard - - var decoded: Manifest - - expect ResultFailure: - decoded = (await erasure.decode(encoded)).tryGet() - - test "Should tolerate losing M (a.k.a row) contiguous data blocks": - const - buffers = 20 - parity = 10 - - let encoded = await encode(buffers, parity) - - # loose M original (systematic) symbols/blocks - for b in 0 ..< (encoded.steps * encoded.ecM): - (await store.delBlock(encoded.treeCid, b)).tryGet() - (await store.delBlock(manifest.treeCid, b)).tryGet() - - discard (await erasure.decode(encoded)).tryGet() - - for d in 0 ..< manifest.blocksCount: - let present = await store.hasBlock(manifest.treeCid, d) - check present.tryGet() - - test "Should tolerate losing M (a.k.a row) contiguous parity blocks": - const - buffers = 20 - parity = 10 - - let - encoded = await encode(buffers, parity) - blocks = collect: - for i in 0 .. encoded.blocksCount: - i - - # loose M parity (all!) symbols/blocks from the dataset - for b in blocks[^(encoded.steps * encoded.ecM) ..^ 1]: - (await store.delBlock(encoded.treeCid, b)).tryGet() - (await store.delBlock(manifest.treeCid, b)).tryGet() - - discard (await erasure.decode(encoded)).tryGet() - - for d in 0 ..< manifest.blocksCount: - let present = await store.hasBlock(manifest.treeCid, d) - check present.tryGet() - - test "Handles edge case of 0 parity blocks": - const - buffers = 20 - parity = 0 - - let encoded = await encode(buffers, parity) - - discard (await erasure.decode(encoded)).tryGet() - - test "Should concurrently encode/decode multiple datasets": - const iterations = 5 - - let - datasetSize = 1.MiBs - ecK = 10.Natural - ecM = 10.Natural - - var encodeTasks = newSeq[Future[?!Manifest]]() - var decodeTasks = newSeq[Future[?!Manifest]]() - var manifests = newSeq[Manifest]() - for i in 0 ..< iterations: - let - # create random data and store it - blockSize = rng.sample(@[1, 2, 4, 8, 16, 32, 64].mapIt(it.KiBs)) - chunker = RandomChunker.new(rng, size = datasetSize, chunkSize = blockSize) - manifest = await storeDataGetManifest(store, chunker) - manifests.add(manifest) - # encode the data concurrently - encodeTasks.add(erasure.encode(manifest, ecK, ecM)) - # wait for all encoding tasks to finish - let encodeResults = await allFinished(encodeTasks) - # decode the data concurrently - for i in 0 ..< encodeResults.len: - decodeTasks.add(erasure.decode(encodeResults[i].read().tryGet())) - # wait for all decoding tasks to finish - let decodeResults = await allFinished(decodeTasks) # TODO: use allFutures - - for j in 0 ..< decodeTasks.len: - let - decoded = decodeResults[j].read().tryGet() - encoded = encodeResults[j].read().tryGet() - check: - decoded.treeCid == manifests[j].treeCid - decoded.treeCid == encoded.originalTreeCid - decoded.blocksCount == encoded.originalBlocksCount - - test "Should handle verifiable manifests": - const - buffers = 20 - parity = 10 - - let - encoded = await encode(buffers, parity) - slotCids = collect(newSeq): - for i in 0 ..< encoded.numSlots: - Cid.example - - verifiable = Manifest.new(encoded, Cid.example, slotCids).tryGet() - - decoded = (await erasure.decode(verifiable)).tryGet() - - check: - decoded.treeCid == manifest.treeCid - decoded.treeCid == verifiable.originalTreeCid - decoded.blocksCount == verifiable.originalBlocksCount - - for i in 1 .. 5: - test "Should encode/decode using various parameters " & $i & "/5": - let - blockSize = rng.sample(@[1, 2, 4, 8, 16, 32, 64].mapIt(it.KiBs)) - datasetSize = 1.MiBs - ecK = 10.Natural - ecM = 10.Natural - - let - chunker = RandomChunker.new(rng, size = datasetSize, chunkSize = blockSize) - manifest = await storeDataGetManifest(store, chunker) - encoded = (await erasure.encode(manifest, ecK, ecM)).tryGet() - decoded = (await erasure.decode(encoded)).tryGet() - - check: - decoded.treeCid == manifest.treeCid - decoded.treeCid == encoded.originalTreeCid - decoded.blocksCount == encoded.originalBlocksCount - - test "Should complete encode/decode task when cancelled": - let - blocksLen = 10000 - parityLen = 10 - data = seq[seq[byte]].new() - chunker = RandomChunker.new( - rng, size = (blocksLen * BlockSize.int), chunkSize = BlockSize - ) - - data[].setLen(blocksLen) - - for i in 0 ..< blocksLen: - let chunk = await chunker.getBytes() - shallowCopy(data[i], @(chunk)) - - let - parity = createDoubleArray(parityLen, BlockSize.int) - paritySeq = seq[seq[byte]].new() - recovered = createDoubleArray(blocksLen, BlockSize.int) - cancelledTaskParity = createDoubleArray(parityLen, BlockSize.int) - cancelledTaskRecovered = createDoubleArray(blocksLen, BlockSize.int) - - paritySeq[].setLen(parityLen) - defer: - freeDoubleArray(parity, parityLen) - freeDoubleArray(cancelledTaskParity, parityLen) - freeDoubleArray(recovered, blocksLen) - freeDoubleArray(cancelledTaskRecovered, blocksLen) - - for i in 0 ..< parityLen: - paritySeq[i] = cast[seq[byte]](parity[i]) - - # call asyncEncode to get the parity - let encFut = - await erasure.asyncEncode(BlockSize.int, blocksLen, parityLen, data, parity) - check encFut.isOk - - let decFut = await erasure.asyncDecode( - BlockSize.int, blocksLen, parityLen, data, paritySeq, recovered - ) - check decFut.isOk - - # call asyncEncode and cancel the task - let encodeFut = erasure.asyncEncode( - BlockSize.int, blocksLen, parityLen, data, cancelledTaskParity - ) - encodeFut.cancel() - - try: - discard await encodeFut - except CatchableError as exc: - check exc of CancelledError - finally: - for i in 0 ..< parityLen: - check equalMem(parity[i], cancelledTaskParity[i], BlockSize.int) - - # call asyncDecode and cancel the task - let decodeFut = erasure.asyncDecode( - BlockSize.int, blocksLen, parityLen, data, paritySeq, cancelledTaskRecovered - ) - decodeFut.cancel() - - try: - discard await decodeFut - except CatchableError as exc: - check exc of CancelledError - finally: - for i in 0 ..< blocksLen: - check equalMem(recovered[i], cancelledTaskRecovered[i], BlockSize.int) diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim index 2077fb81..82c24761 100644 --- a/tests/codex/testlogutils.nim +++ b/tests/codex/testlogutils.nim @@ -4,9 +4,7 @@ import std/strutils import pkg/unittest2 import pkg/codex/blocktype import pkg/codex/conf -import pkg/codex/contracts/requests import pkg/codex/logutils -import pkg/codex/purchasing/purchaseid import pkg/codex/units import pkg/codex/utils/json import pkg/libp2p/cid @@ -200,63 +198,6 @@ checksuite "Test logging output": check logged("int", "123") check loggedJson("int", "123") - test "logs EthAddress correctly": - let address = EthAddress.fromHex("0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b") - log address - check logged("address", "0xf75e..1b1b") - check loggedJson("address", "\"0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b\"") - - test "logs PurchaseId correctly": - let id = PurchaseId.fromHex( - "0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea" - ) - log id - check logged("id", "0x7120..3dea") - - test "logs RequestId correctly": - let id = RequestId.fromHex( - "0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea" - ) - log id - check logged("id", "0x7120..3dea") - check loggedJson( - "id", "\"0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea\"" - ) - - test "logs seq[RequestId] correctly": - let id = RequestId.fromHex( - "0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea" - ) - let id2 = RequestId.fromHex( - "0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282" - ) - let ids = @[id, id2] - log ids - check logged("ids", "\"@[0x7120..3dea, 0x9ab2..c282]\"") - check loggedJson( - "ids", - """["0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea","0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282"]""", - ) - - test "logs SlotId correctly": - let id = SlotId.fromHex( - "0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282" - ) - log id - check logged("id", "0x9ab2..c282") - check loggedJson( - "id", "\"0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282\"" - ) - - test "logs Nonce correctly": - let n = - Nonce.fromHex("ce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea") - log n - check logged("n", "0xce88..d6ea") - check loggedJson( - "n", "\"0xce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea\"" - ) - test "logs MultiAddress correctly": let ma = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet log ma diff --git a/tests/codex/testmanifest.nim b/tests/codex/testmanifest.nim index ea9465d5..82583b31 100644 --- a/tests/codex/testmanifest.nim +++ b/tests/codex/testmanifest.nim @@ -5,7 +5,6 @@ import pkg/codex/blocktype as bt import pkg/codex/manifest import pkg/poseidon2 -import pkg/codex/slots import pkg/codex/merkletree import pkg/codex/indexingstrategy @@ -27,21 +26,6 @@ suite "Manifest": strategy = SteppedStrategy, ) - leaves = [ - 0.toF.Poseidon2Hash, 1.toF.Poseidon2Hash, 2.toF.Poseidon2Hash, 3.toF.Poseidon2Hash - ] - - slotLeavesCids = leaves.toSlotCids().tryGet - - tree = Poseidon2Tree.init(leaves).tryGet - verifyCid = tree.root.tryGet.toVerifyCid().tryGet - - verifiableManifest = Manifest - .new( - manifest = protectedManifest, verifyRoot = verifyCid, slotRoots = slotLeavesCids - ) - .tryGet() - proc encodeDecode(manifest: Manifest): Manifest = let e = manifest.encode().tryGet() Manifest.decode(e).tryGet() @@ -59,63 +43,3 @@ suite "Manifest": check: encodeDecode(large) == large - - test "Should encode/decode to/from protected manifest": - check: - encodeDecode(protectedManifest) == protectedManifest - - test "Should encode/decode to/from verifiable manifest": - check: - encodeDecode(verifiableManifest) == verifiableManifest - -suite "Manifest - Attribute Inheritance": - proc makeProtectedManifest(strategy: StrategyType): Manifest = - Manifest.new( - manifest = Manifest.new( - treeCid = Cid.example, - blockSize = 1.MiBs, - datasetSize = 100.MiBs, - filename = "codex.png".some, - mimetype = "image/png".some, - ), - treeCid = Cid.example, - datasetSize = 200.MiBs, - ecK = 1, - ecM = 1, - strategy = strategy, - ) - - test "Should preserve interleaving strategy for protected manifest in verifiable manifest": - var verifiable = Manifest - .new( - manifest = makeProtectedManifest(SteppedStrategy), - verifyRoot = Cid.example, - slotRoots = @[Cid.example, Cid.example], - ) - .tryGet() - - check verifiable.protectedStrategy == SteppedStrategy - - verifiable = Manifest - .new( - manifest = makeProtectedManifest(LinearStrategy), - verifyRoot = Cid.example, - slotRoots = @[Cid.example, Cid.example], - ) - .tryGet() - - check verifiable.protectedStrategy == LinearStrategy - - test "Should preserve metadata for manifest in verifiable manifest": - var verifiable = Manifest - .new( - manifest = makeProtectedManifest(SteppedStrategy), - verifyRoot = Cid.example, - slotRoots = @[Cid.example, Cid.example], - ) - .tryGet() - - check verifiable.filename.isSome == true - check verifiable.filename.get() == "codex.png" - check verifiable.mimetype.isSome == true - check verifiable.mimetype.get() == "image/png" diff --git a/tests/codex/testnode.nim b/tests/codex/testnode.nim index c6f0154b..54c9a21f 100644 --- a/tests/codex/testnode.nim +++ b/tests/codex/testnode.nim @@ -1,5 +1,3 @@ import ./node/testnode -import ./node/testcontracts -import ./node/testslotrepair {.warning[UnusedImport]: off.} diff --git a/tests/codex/testpurchasing.nim b/tests/codex/testpurchasing.nim deleted file mode 100644 index 339daa18..00000000 --- a/tests/codex/testpurchasing.nim +++ /dev/null @@ -1,253 +0,0 @@ -import std/times -import pkg/chronos -import pkg/stint -import pkg/codex/purchasing -import pkg/codex/purchasing/states/finished -import pkg/codex/purchasing/states/started -import pkg/codex/purchasing/states/submitted -import pkg/codex/purchasing/states/unknown -import pkg/codex/purchasing/states/cancelled -import pkg/codex/purchasing/states/failed - -import ../asynctest -import ./helpers/mockmarket -import ./helpers/mockclock -import ./examples -import ./helpers - -asyncchecksuite "Purchasing": - var purchasing: Purchasing - var market: MockMarket - var clock: MockClock - var request, populatedRequest: StorageRequest - - setup: - market = MockMarket.new() - clock = MockClock.new() - purchasing = Purchasing.new(market, clock) - request = StorageRequest( - ask: StorageAsk( - slots: uint8.example.uint64, - slotSize: uint32.example.uint64, - duration: uint16.example.uint64, - pricePerBytePerSecond: uint8.example.u256, - ) - ) - - # We need request which has stable ID during the whole Purchasing pipeline - # for some tests (related to expiry). Because of Purchasing.populate() we need - # to do the steps bellow. - populatedRequest = StorageRequest.example - populatedRequest.client = await market.getSigner() - - test "submits a storage request when asked": - discard await purchasing.purchase(request) - check eventually market.requested.len > 0 - check market.requested[0].ask.slots == request.ask.slots - check market.requested[0].ask.slotSize == request.ask.slotSize - check market.requested[0].ask.duration == request.ask.duration - check market.requested[0].ask.pricePerBytePerSecond == - request.ask.pricePerBytePerSecond - - test "remembers purchases": - let purchase1 = await purchasing.purchase(request) - let purchase2 = await purchasing.purchase(request) - check purchasing.getPurchase(purchase1.id) == some purchase1 - check purchasing.getPurchase(purchase2.id) == some purchase2 - - test "has a default value for proof probability": - check purchasing.proofProbability != 0.u256 - - test "can change default value for proof probability": - purchasing.proofProbability = 42.u256 - discard await purchasing.purchase(request) - check eventually market.requested.len > 0 - check market.requested[0].ask.proofProbability == 42.u256 - - test "can override proof probability per request": - request.ask.proofProbability = 42.u256 - discard await purchasing.purchase(request) - check eventually market.requested.len > 0 - check market.requested[0].ask.proofProbability == 42.u256 - - test "includes a random nonce in every storage request": - discard await purchasing.purchase(request) - discard await purchasing.purchase(request) - check eventually market.requested.len > 0 - check market.requested[0].nonce != market.requested[1].nonce - - test "sets client address in request": - discard await purchasing.purchase(request) - check eventually market.requested.len > 0 - check market.requested[0].client == await market.getSigner() - - test "succeeds when request is finished": - market.requestExpiry[populatedRequest.id] = getTime().toUnix() + 10 - let purchase = await purchasing.purchase(populatedRequest) - - check eventually market.requested.len > 0 - let request = market.requested[0] - let requestEnd = getTime().toUnix() + 42 - market.requestEnds[request.id] = requestEnd - - market.emitRequestFulfilled(request.id) - clock.set(requestEnd + 1) - await purchase.wait() - check purchase.error.isNone - - test "fails when request times out": - let purchase = await purchasing.purchase(populatedRequest) - check eventually market.requested.len > 0 - - let expiry = market.requestExpiry[populatedRequest.id] - clock.set(expiry + 1) - expect PurchaseTimeout: - await purchase.wait() - - test "checks that funds were withdrawn when purchase times out": - let purchase = await purchasing.purchase(populatedRequest) - check eventually market.requested.len > 0 - let request = market.requested[0] - let expiry = market.requestExpiry[populatedRequest.id] - clock.set(expiry + 1) - expect PurchaseTimeout: - await purchase.wait() - check market.withdrawn == @[request.id] - -suite "Purchasing state machine": - var purchasing: Purchasing - var market: MockMarket - var clock: MockClock - var request: StorageRequest - - setup: - market = MockMarket.new() - clock = MockClock.new() - purchasing = Purchasing.new(market, clock) - request = StorageRequest( - ask: StorageAsk( - slots: uint8.example.uint64, - slotSize: uint32.example.uint64, - duration: uint16.example.uint64, - pricePerBytePerSecond: uint8.example.u256, - ) - ) - - test "loads active purchases from market": - let me = await market.getSigner() - let request1, request2, request3 = StorageRequest.example - market.requested = @[request1, request2, request3] - market.activeRequests[me] = @[request1.id, request2.id] - await purchasing.load() - check isSome purchasing.getPurchase(PurchaseId(request1.id)) - check isSome purchasing.getPurchase(PurchaseId(request2.id)) - check isNone purchasing.getPurchase(PurchaseId(request3.id)) - - test "loads correct purchase.future state for purchases from market": - let me = await market.getSigner() - let request1, request2, request3, request4, request5 = StorageRequest.example - market.requested = @[request1, request2, request3, request4, request5] - market.activeRequests[me] = - @[request1.id, request2.id, request3.id, request4.id, request5.id] - market.requestState[request1.id] = RequestState.New - market.requestState[request2.id] = RequestState.Started - market.requestState[request3.id] = RequestState.Cancelled - market.requestState[request4.id] = RequestState.Finished - market.requestState[request5.id] = RequestState.Failed - - # ensure the started state doesn't error, giving a false positive test result - market.requestEnds[request2.id] = clock.now() - 1 - - await purchasing.load() - check eventually purchasing.getPurchase(PurchaseId(request1.id)) .? finished == - false.some - check eventually purchasing.getPurchase(PurchaseId(request2.id)) .? finished == - true.some - check eventually purchasing.getPurchase(PurchaseId(request3.id)) .? finished == - true.some - check eventually purchasing.getPurchase(PurchaseId(request4.id)) .? finished == - true.some - check eventually purchasing.getPurchase(PurchaseId(request5.id)) .? finished == - true.some - check eventually purchasing.getPurchase(PurchaseId(request5.id)) .? error.isSome - - test "moves to PurchaseSubmitted when request state is New": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - market.requested = @[request] - market.requestState[request.id] = RequestState.New - let next = await PurchaseUnknown().run(purchase) - check !next of PurchaseSubmitted - - test "moves to PurchaseStarted when request state is Started": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - market.requestEnds[request.id] = clock.now() + request.ask.duration.int64 - market.requested = @[request] - market.requestState[request.id] = RequestState.Started - let next = await PurchaseUnknown().run(purchase) - check !next of PurchaseStarted - - test "moves to PurchaseCancelled when request state is Cancelled": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - market.requested = @[request] - market.requestState[request.id] = RequestState.Cancelled - let next = await PurchaseUnknown().run(purchase) - check !next of PurchaseCancelled - - test "moves to PurchaseFinished when request state is Finished": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - market.requested = @[request] - market.requestState[request.id] = RequestState.Finished - let next = await PurchaseUnknown().run(purchase) - check !next of PurchaseFinished - - test "moves to PurchaseFailed when request state is Failed": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - market.requested = @[request] - market.requestState[request.id] = RequestState.Failed - let next = await PurchaseUnknown().run(purchase) - check !next of PurchaseFailed - - test "moves to PurchaseFailed state once RequestFailed emitted": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - market.requestEnds[request.id] = clock.now() + request.ask.duration.int64 - let future = PurchaseStarted().run(purchase) - - market.emitRequestFailed(request.id) - - let next = await future - check !next of PurchaseFailed - - test "moves to PurchaseFinished state once request finishes": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - market.requestEnds[request.id] = clock.now() + request.ask.duration.int64 - let future = PurchaseStarted().run(purchase) - - clock.advance(request.ask.duration.int64 + 1) - - let next = await future - check !next of PurchaseFinished - - test "withdraw funds in PurchaseFinished": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - discard await PurchaseFinished().run(purchase) - check request.id in market.withdrawn - - test "withdraw funds in PurchaseFailed": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - discard await PurchaseFailed().run(purchase) - check request.id in market.withdrawn - - test "withdraw funds in PurchaseCancelled": - let request = StorageRequest.example - let purchase = Purchase.new(request, market, clock) - discard await PurchaseCancelled().run(purchase) - check request.id in market.withdrawn diff --git a/tests/codex/testsales.nim b/tests/codex/testsales.nim deleted file mode 100644 index 38c530f6..00000000 --- a/tests/codex/testsales.nim +++ /dev/null @@ -1,7 +0,0 @@ -import ./sales/testsales -import ./sales/teststates -import ./sales/testreservations -import ./sales/testslotqueue -import ./sales/testsalesagent - -{.warning[UnusedImport]: off.} diff --git a/tests/codex/testslots.nim b/tests/codex/testslots.nim deleted file mode 100644 index 059de7c2..00000000 --- a/tests/codex/testslots.nim +++ /dev/null @@ -1,8 +0,0 @@ -import ./slots/testslotbuilder -import ./slots/testsampler -import ./slots/testconverters -import ./slots/testbackends -import ./slots/testprover -import ./slots/testbackendfactory - -{.warning[UnusedImport]: off.} diff --git a/tests/codex/testvalidation.nim b/tests/codex/testvalidation.nim deleted file mode 100644 index edb3d0f2..00000000 --- a/tests/codex/testvalidation.nim +++ /dev/null @@ -1,229 +0,0 @@ -import pkg/chronos -import std/strformat -import std/times - -import codex/validation -import codex/periods -import codex/clock - -import ../asynctest -import ./helpers/mockmarket -import ./helpers/mockclock -import ./examples -import ./helpers - -logScope: - topics = "testValidation" - -asyncchecksuite "validation": - let period = 10.uint64 - let timeout = 5.uint64 - let maxSlots = MaxSlots(100) - let validationGroups = ValidationGroups(8).some - let slot = Slot.example - let proof = Groth16Proof.example - let collateral = slot.request.ask.collateralPerSlot - - var market: MockMarket - var clock: MockClock - var groupIndex: uint16 - var validation: Validation - - proc initValidationConfig( - maxSlots: MaxSlots, validationGroups: ?ValidationGroups, groupIndex: uint16 = 0 - ): ValidationConfig = - without validationConfig =? - ValidationConfig.init(maxSlots, groups = validationGroups, groupIndex), error: - raiseAssert fmt"Creating ValidationConfig failed! Error msg: {error.msg}" - validationConfig - - proc newValidation( - clock: Clock, - market: Market, - maxSlots: MaxSlots, - validationGroups: ?ValidationGroups, - groupIndex: uint16 = 0, - ): Validation = - let validationConfig = initValidationConfig(maxSlots, validationGroups, groupIndex) - Validation.new(clock, market, validationConfig) - - setup: - groupIndex = groupIndexForSlotId(slot.id, !validationGroups) - clock = MockClock.new() - market = MockMarket.new(clock) - market.config.proofs.period = period - market.config.proofs.timeout = timeout - validation = newValidation(clock, market, maxSlots, validationGroups, groupIndex) - - teardown: - # calling stop on validation that did not start is harmless - await validation.stop() - - proc advanceToNextPeriod() = - let periodicity = Periodicity(seconds: period) - let period = periodicity.periodOf(clock.now().Timestamp) - let periodEnd = periodicity.periodEnd(period) - clock.set(periodEnd.toSecondsSince1970 + 1) - - test "the list of slots that it's monitoring is empty initially": - check validation.slots.len == 0 - - for (validationGroups, groupIndex) in [(100, 100'u16), (100, 101'u16)]: - test "initializing ValidationConfig fails when groupIndex is " & - "greater than or equal to validationGroups " & - fmt"(testing for {groupIndex = }, {validationGroups = })": - let groups = ValidationGroups(validationGroups).some - let validationConfig = - ValidationConfig.init(maxSlots, groups = groups, groupIndex = groupIndex) - check validationConfig.isFailure == true - check validationConfig.error.msg == - "The value of the group index " & "must be less than validation groups! " & - fmt"(got: {groupIndex = }, groups = {!groups})" - - test "initializing ValidationConfig fails when maxSlots is negative": - let maxSlots = -1 - let validationConfig = - ValidationConfig.init(maxSlots = maxSlots, groups = ValidationGroups.none) - check validationConfig.isFailure == true - check validationConfig.error.msg == - "The value of maxSlots must " & - fmt"be greater than or equal to 0! (got: {maxSlots})" - - test "initializing ValidationConfig fails when maxSlots is negative " & - "(validationGroups set)": - let maxSlots = -1 - let groupIndex = 0'u16 - let validationConfig = - ValidationConfig.init(maxSlots = maxSlots, groups = validationGroups, groupIndex) - check validationConfig.isFailure == true - check validationConfig.error.msg == - "The value of maxSlots must " & - fmt"be greater than or equal to 0! (got: {maxSlots})" - - test "slot is not observed if it is not in the validation group": - validation = newValidation( - clock, - market, - maxSlots, - validationGroups, - (groupIndex + 1) mod uint16(!validationGroups), - ) - await validation.start() - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - check validation.slots.len == 0 - - test "when a slot is filled on chain, it is added to the list": - await validation.start() - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - check validation.slots == @[slot.id] - - test "slot should be observed if maxSlots is set to 0": - validation = newValidation(clock, market, maxSlots = 0, ValidationGroups.none) - await validation.start() - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - check validation.slots == @[slot.id] - - test "slot should be observed if validation group is not set (and " & - "maxSlots is not 0)": - validation = newValidation(clock, market, maxSlots, ValidationGroups.none) - await validation.start() - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - check validation.slots == @[slot.id] - - for state in [SlotState.Finished, SlotState.Failed]: - test fmt"when slot state changes to {state}, it is removed from the list": - validation = newValidation(clock, market, maxSlots, validationGroups) - await validation.start() - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - market.slotState[slot.id] = state - advanceToNextPeriod() - check eventually validation.slots.len == 0 - - test "when a proof is missed, it is marked as missing": - await validation.start() - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - market.setCanMarkProofAsMissing(slot.id, true) - advanceToNextPeriod() - await sleepAsync(100.millis) # allow validation loop to run - check market.markedAsMissingProofs.contains(slot.id) - - test "when a proof can not be marked as missing, it will not be marked": - await validation.start() - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - market.setCanMarkProofAsMissing(slot.id, false) - advanceToNextPeriod() - await sleepAsync(100.millis) # allow validation loop to run - check market.markedAsMissingProofs.len == 0 - - test "it does not monitor more than the maximum number of slots": - validation = newValidation(clock, market, maxSlots, ValidationGroups.none) - await validation.start() - for _ in 0 ..< maxSlots + 1: - let slot = Slot.example - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - check validation.slots.len == maxSlots - - suite "restoring historical state": - test "it retrieves the historical state " & "for max 30 days in the past": - let earlySlot = Slot.example - await market.fillSlot( - earlySlot.request.id, earlySlot.slotIndex, proof, collateral - ) - let fromTime = clock.now() - clock.set(fromTime + 1) - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - - let duration: times.Duration = initDuration(days = 30) - clock.set(fromTime + duration.inSeconds + 1) - - validation = newValidation(clock, market, maxSlots = 0, ValidationGroups.none) - await validation.start() - - check validation.slots == @[slot.id] - - for state in [SlotState.Finished, SlotState.Failed]: - test "when restoring historical state, " & fmt"it excludes slots in {state} state": - let slot1 = Slot.example - let slot2 = Slot.example - await market.fillSlot(slot1.request.id, slot1.slotIndex, proof, collateral) - await market.fillSlot(slot2.request.id, slot2.slotIndex, proof, collateral) - - market.slotState[slot1.id] = state - - validation = newValidation(clock, market, maxSlots = 0, ValidationGroups.none) - await validation.start() - - check validation.slots == @[slot2.id] - - test "it does not monitor more than the maximum number of slots ": - for _ in 0 ..< maxSlots + 1: - let slot = Slot.example - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - validation = newValidation(clock, market, maxSlots, ValidationGroups.none) - await validation.start() - check validation.slots.len == maxSlots - - test "slot is not observed if it is not in the validation group": - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - validation = newValidation( - clock, - market, - maxSlots, - validationGroups, - (groupIndex + 1) mod uint16(!validationGroups), - ) - await validation.start() - check validation.slots.len == 0 - - test "slot should be observed if maxSlots is set to 0": - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - validation = newValidation(clock, market, maxSlots = 0, ValidationGroups.none) - await validation.start() - check validation.slots == @[slot.id] - - test "slot should be observed if validation " & - "group is not set (and maxSlots is not 0)": - await market.fillSlot(slot.request.id, slot.slotIndex, proof, collateral) - validation = newValidation(clock, market, maxSlots, ValidationGroups.none) - await validation.start() - check validation.slots == @[slot.id] diff --git a/tests/contracts/deployment.nim b/tests/contracts/deployment.nim deleted file mode 100644 index f45aa625..00000000 --- a/tests/contracts/deployment.nim +++ /dev/null @@ -1,19 +0,0 @@ -import std/os -import std/options -import pkg/ethers -import pkg/codex/contracts/marketplace - -const hardhatMarketAddress = - Address.init("0x322813Fd9A801c5507c9de605d63CEA4f2CE6c44").get() -const hardhatMarketWithDummyVerifier = - Address.init("0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f").get() -const marketAddressEnvName = "CODEX_MARKET_ADDRESS" - -proc address*(_: type Marketplace, dummyVerifier = false): Address = - if existsEnv(marketAddressEnvName): - without address =? Address.init(getEnv(marketAddressEnvName)): - raiseAssert "Invalid env. variable marketplace contract address" - - return address - - if dummyVerifier: hardhatMarketWithDummyVerifier else: hardhatMarketAddress diff --git a/tests/contracts/examples.nim b/tests/contracts/examples.nim deleted file mode 100644 index 4e9475d0..00000000 --- a/tests/contracts/examples.nim +++ /dev/null @@ -1,7 +0,0 @@ -import pkg/ethers -import ../examples - -export examples - -proc example*(_: type Address): Address = - Address(array[20, byte].example) diff --git a/tests/contracts/helpers/mockprovider.nim b/tests/contracts/helpers/mockprovider.nim deleted file mode 100644 index c5be8ad7..00000000 --- a/tests/contracts/helpers/mockprovider.nim +++ /dev/null @@ -1,82 +0,0 @@ -import std/strutils -import std/tables - -import pkg/ethers/provider -from codex/clock import SecondsSince1970 - -export provider.Block - -type MockProvider* = ref object of Provider - blocks: OrderedTableRef[int, Block] - earliest: ?int - latest: ?int - -method getBlock*( - provider: MockProvider, tag: BlockTag -): Future[?Block] {.async: (raises: [ProviderError, CancelledError]).} = - try: - if tag == BlockTag.latest: - if latestBlock =? provider.latest: - if provider.blocks.hasKey(latestBlock): - return provider.blocks[latestBlock].some - elif tag == BlockTag.earliest: - if earliestBlock =? provider.earliest: - if provider.blocks.hasKey(earliestBlock): - return provider.blocks[earliestBlock].some - elif tag == BlockTag.pending: - raiseAssert "MockProvider does not yet support BlockTag.pending" - else: - let blockNumber = parseHexInt($tag) - if provider.blocks.hasKey(blockNumber): - return provider.blocks[blockNumber].some - return Block.none - except: - return Block.none - -proc updateEarliestAndLatest(provider: MockProvider, blockNumber: int) = - if provider.earliest.isNone: - provider.earliest = blockNumber.some - provider.latest = blockNumber.some - -proc addBlocks*(provider: MockProvider, blocks: OrderedTableRef[int, Block]) = - for number, blk in blocks.pairs: - if provider.blocks.hasKey(number): - continue - provider.updateEarliestAndLatest(number) - provider.blocks[number] = blk - -proc addBlock*(provider: MockProvider, number: int, blk: Block) = - if not provider.blocks.hasKey(number): - provider.updateEarliestAndLatest(number) - provider.blocks[number] = blk - -proc newMockProvider*(): MockProvider = - MockProvider( - blocks: newOrderedTable[int, Block](), earliest: int.none, latest: int.none - ) - -proc newMockProvider*(blocks: OrderedTableRef[int, Block]): MockProvider = - let provider = newMockProvider() - provider.addBlocks(blocks) - provider - -proc newMockProvider*( - numberOfBlocks: int, - earliestBlockNumber: int, - earliestBlockTimestamp: SecondsSince1970, - timeIntervalBetweenBlocks: SecondsSince1970, -): MockProvider = - var blocks = newOrderedTable[int, provider.Block]() - var blockNumber = earliestBlockNumber - var blockTime = earliestBlockTimestamp - for i in 0 ..< numberOfBlocks: - blocks[blockNumber] = provider.Block( - number: blockNumber.u256.some, timestamp: blockTime.u256, hash: BlockHash.none - ) - inc blockNumber - inc blockTime, timeIntervalBetweenBlocks.int - MockProvider( - blocks: blocks, - earliest: earliestBlockNumber.some, - latest: (earliestBlockNumber + numberOfBlocks - 1).some, - ) diff --git a/tests/contracts/nim.cfg b/tests/contracts/nim.cfg deleted file mode 100644 index 1c2f0c13..00000000 --- a/tests/contracts/nim.cfg +++ /dev/null @@ -1 +0,0 @@ ---path:"../.." diff --git a/tests/contracts/testClock.nim b/tests/contracts/testClock.nim deleted file mode 100644 index 4c3ad03c..00000000 --- a/tests/contracts/testClock.nim +++ /dev/null @@ -1,50 +0,0 @@ -import std/times -import pkg/chronos -import codex/contracts/clock -import codex/utils/json -import ../ethertest - -ethersuite "On-Chain Clock": - var clock: OnChainClock - - setup: - clock = OnChainClock.new(ethProvider) - await clock.start() - - teardown: - await clock.stop() - - test "returns the current time of the EVM": - let latestBlock = (!await ethProvider.getBlock(BlockTag.latest)) - let timestamp = latestBlock.timestamp.truncate(int64) - check clock.now() == timestamp - - test "updates time with timestamp of new blocks": - let future = (getTime() + 42.years).toUnix - discard await ethProvider.send("evm_setNextBlockTimestamp", @[%future]) - discard await ethProvider.send("evm_mine") - - # Ensure that state updates are sync with WS - discard await ethProvider.getBlock(BlockTag.latest) - - check eventually clock.now() == future - - test "can wait until a certain time is reached by the chain": - let future = clock.now() + 42 # seconds - let waiting = clock.waitUntil(future) - discard await ethProvider.send("evm_setNextBlockTimestamp", @[%future]) - discard await ethProvider.send("evm_mine") - check await waiting.withTimeout(chronos.milliseconds(500)) - - test "can wait until a certain time is reached by the wall-clock": - let future = clock.now() + 1 # seconds - let waiting = clock.waitUntil(future) - check await waiting.withTimeout(chronos.seconds(2)) - - test "handles starting multiple times": - await clock.start() - await clock.start() - - test "handles stopping multiple times": - await clock.stop() - await clock.stop() diff --git a/tests/contracts/testContracts.nim b/tests/contracts/testContracts.nim deleted file mode 100644 index 84708ecd..00000000 --- a/tests/contracts/testContracts.nim +++ /dev/null @@ -1,132 +0,0 @@ -import pkg/chronos -import pkg/ethers/erc20 -import codex/contracts -import ../ethertest -import ./examples -import ./time -import ./deployment - -ethersuite "Marketplace contracts": - let proof = Groth16Proof.example - - var client, host: Signer - var rewardRecipient, collateralRecipient: Address - var marketplace: Marketplace - var token: Erc20Token - var periodicity: Periodicity - var request: StorageRequest - var slotId: SlotId - var filledAt: UInt256 - - proc expectedPayout(endTimestamp: UInt256): UInt256 = - return (endTimestamp - filledAt) * request.ask.pricePerSlotPerSecond() - - proc switchAccount(account: Signer) = - marketplace = marketplace.connect(account) - token = token.connect(account) - - setup: - client = ethProvider.getSigner(accounts[0]) - host = ethProvider.getSigner(accounts[1]) - rewardRecipient = accounts[2] - collateralRecipient = accounts[3] - - let address = Marketplace.address(dummyVerifier = true) - marketplace = Marketplace.new(address, ethProvider.getSigner()) - - let tokenAddress = await marketplace.token() - token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) - - let config = await marketplace.configuration() - periodicity = Periodicity(seconds: config.proofs.period) - - request = StorageRequest.example - request.client = await client.getAddress() - - switchAccount(client) - discard await token.approve(marketplace.address, request.totalPrice).confirm(1) - discard await marketplace.requestStorage(request).confirm(1) - switchAccount(host) - discard - await token.approve(marketplace.address, request.ask.collateralPerSlot).confirm(1) - discard await marketplace.reserveSlot(request.id, 0.uint64).confirm(1) - let receipt = await marketplace.fillSlot(request.id, 0.uint64, proof).confirm(1) - filledAt = await ethProvider.blockTime(BlockTag.init(!receipt.blockNumber)) - slotId = request.slotId(0.uint64) - - proc waitUntilProofRequired(slotId: SlotId) {.async.} = - let currentPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await ethProvider.advanceTimeTo(periodicity.periodEnd(currentPeriod).u256) - while not ( - (await marketplace.isProofRequired(slotId)) and - (await marketplace.getPointer(slotId)) < 250 - ) - : - await ethProvider.advanceTime(periodicity.seconds.u256) - - proc startContract() {.async.} = - for slotIndex in 1 ..< request.ask.slots: - discard await token - .approve(marketplace.address, request.ask.collateralPerSlot) - .confirm(1) - discard await marketplace.reserveSlot(request.id, slotIndex.uint64).confirm(1) - discard await marketplace.fillSlot(request.id, slotIndex.uint64, proof).confirm(1) - - test "accept marketplace proofs": - switchAccount(host) - await waitUntilProofRequired(slotId) - discard await marketplace.submitProof(slotId, proof).confirm(1) - - test "can mark missing proofs": - switchAccount(host) - await waitUntilProofRequired(slotId) - let missingPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - let endOfPeriod = periodicity.periodEnd(missingPeriod) - await ethProvider.advanceTimeTo(endOfPeriod.u256 + 1) - switchAccount(client) - discard await marketplace.markProofAsMissing(slotId, missingPeriod).confirm(1) - - test "can be paid out at the end": - switchAccount(host) - let address = await host.getAddress() - await startContract() - let requestEnd = await marketplace.requestEnd(request.id) - await ethProvider.advanceTimeTo(requestEnd.u256 + 1) - let startBalance = await token.balanceOf(address) - discard await marketplace.freeSlot(slotId).confirm(1) - let endBalance = await token.balanceOf(address) - check endBalance == - (startBalance + expectedPayout(requestEnd.u256) + request.ask.collateralPerSlot) - - test "can be paid out at the end, specifying reward and collateral recipient": - switchAccount(host) - let hostAddress = await host.getAddress() - await startContract() - let requestEnd = await marketplace.requestEnd(request.id) - await ethProvider.advanceTimeTo(requestEnd.u256 + 1) - let startBalanceHost = await token.balanceOf(hostAddress) - let startBalanceReward = await token.balanceOf(rewardRecipient) - let startBalanceCollateral = await token.balanceOf(collateralRecipient) - discard await marketplace - .freeSlot(slotId, rewardRecipient, collateralRecipient) - .confirm(1) - let endBalanceHost = await token.balanceOf(hostAddress) - let endBalanceReward = await token.balanceOf(rewardRecipient) - let endBalanceCollateral = await token.balanceOf(collateralRecipient) - - check endBalanceHost == startBalanceHost - check endBalanceReward == (startBalanceReward + expectedPayout(requestEnd.u256)) - check endBalanceCollateral == - (startBalanceCollateral + request.ask.collateralPerSlot) - - test "cannot mark proofs missing for cancelled request": - let expiry = await marketplace.requestExpiry(request.id) - await ethProvider.advanceTimeTo((expiry + 1).u256) - switchAccount(client) - let missingPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await ethProvider.advanceTime(periodicity.seconds.u256) - expect Marketplace_SlotNotAcceptingProofs: - discard await marketplace.markProofAsMissing(slotId, missingPeriod).confirm(1) diff --git a/tests/contracts/testDeployment.nim b/tests/contracts/testDeployment.nim deleted file mode 100644 index e8983437..00000000 --- a/tests/contracts/testDeployment.nim +++ /dev/null @@ -1,43 +0,0 @@ -import pkg/ethers -import pkg/questionable -import codex/contracts/deployment -import codex/contracts - -import ../asynctest -import ../checktest - -type MockProvider = ref object of Provider - chainId*: UInt256 - -method getChainId*( - provider: MockProvider -): Future[UInt256] {.async: (raises: [ProviderError, CancelledError]).} = - return provider.chainId - -asyncchecksuite "Deployment": - let provider = MockProvider() - - test "uses conf value as priority": - let deployment = Deployment.new( - provider, some !Address.init("0x59b670e9fA9D0A427751Af201D676719a970aaaa") - ) - provider.chainId = 1.u256 - - let address = await deployment.address(Marketplace) - check address.isSome - check $(!address) == "0x59b670e9fa9d0a427751af201d676719a970aaaa" - - test "uses chainId hardcoded values as fallback": - let deployment = Deployment.new(provider) - provider.chainId = 167005.u256 - - let address = await deployment.address(Marketplace) - check address.isSome - check $(!address) == "0x948cf9291b77bd7ad84781b9047129addf1b894f" - - test "return none for unknown networks": - let deployment = Deployment.new(provider) - provider.chainId = 1.u256 - - let address = await deployment.address(Marketplace) - check address.isNone diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim deleted file mode 100644 index cf4882ab..00000000 --- a/tests/contracts/testMarket.nim +++ /dev/null @@ -1,681 +0,0 @@ -import std/options -import std/importutils -import pkg/chronos -import pkg/ethers/erc20 -import codex/contracts -import pkg/libp2p/cid -import pkg/lrucache -import ../ethertest -import ./examples -import ./time -import ./deployment - -privateAccess(OnChainMarket) # enable access to private fields - -# to see supportive information in the test output -# use `-d:"chronicles_enabled_topics:testMarket:DEBUG` option -# when compiling the test file -logScope: - topics = "testMarket" - -ethersuite "On-Chain Market": - let proof = Groth16Proof.example - - var market: OnChainMarket - var marketplace: Marketplace - var token: Erc20Token - var request: StorageRequest - var slotIndex: uint64 - var periodicity: Periodicity - var host: Signer - var otherHost: Signer - var hostRewardRecipient: Address - - proc expectedPayout( - r: StorageRequest, startTimestamp: UInt256, endTimestamp: UInt256 - ): UInt256 = - return (endTimestamp - startTimestamp) * r.ask.pricePerSlotPerSecond - - proc switchAccount(account: Signer) = - marketplace = marketplace.connect(account) - token = token.connect(account) - market = OnChainMarket.new(marketplace, market.rewardRecipient) - - setup: - let address = Marketplace.address(dummyVerifier = true) - marketplace = Marketplace.new(address, ethProvider.getSigner()) - let config = await marketplace.configuration() - hostRewardRecipient = accounts[2] - - market = OnChainMarket.new(marketplace) - let tokenAddress = await marketplace.token() - token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) - - periodicity = Periodicity(seconds: config.proofs.period) - - request = StorageRequest.example - request.client = accounts[0] - host = ethProvider.getSigner(accounts[1]) - otherHost = ethProvider.getSigner(accounts[3]) - - slotIndex = request.ask.slots div 2 - - proc advanceToNextPeriod() {.async.} = - let currentPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await ethProvider.advanceTimeTo((periodicity.periodEnd(currentPeriod) + 1).u256) - - proc advanceToCancelledRequest(request: StorageRequest) {.async.} = - let expiry = (await market.requestExpiresAt(request.id)) + 1 - await ethProvider.advanceTimeTo(expiry.u256) - - proc waitUntilProofRequired(slotId: SlotId) {.async.} = - await advanceToNextPeriod() - while not ( - (await marketplace.isProofRequired(slotId)) and - (await marketplace.getPointer(slotId)) < 250 - ) - : - await advanceToNextPeriod() - - test "caches marketplace configuration": - check isNone market.configuration - discard await market.periodicity() - check isSome market.configuration - - test "fails to instantiate when contract does not have a signer": - let storageWithoutSigner = marketplace.connect(ethProvider) - expect AssertionDefect: - discard OnChainMarket.new(storageWithoutSigner) - - test "knows signer address": - check (await market.getSigner()) == (await ethProvider.getSigner().getAddress()) - - test "can retrieve proof periodicity": - let periodicity = await market.periodicity() - let config = await marketplace.configuration() - let periodLength = config.proofs.period - check periodicity.seconds == periodLength - - test "can retrieve proof timeout": - let proofTimeout = await market.proofTimeout() - let config = await marketplace.configuration() - check proofTimeout == config.proofs.timeout - - test "supports marketplace requests": - await market.requestStorage(request) - - test "can retrieve previously submitted requests": - check (await market.getRequest(request.id)) == none StorageRequest - await market.requestStorage(request) - let r = await market.getRequest(request.id) - check (r) == some request - - test "withdraws funds to client": - let clientAddress = request.client - - await market.requestStorage(request) - await advanceToCancelledRequest(request) - let startBalanceClient = await token.balanceOf(clientAddress) - await market.withdrawFunds(request.id) - - let endBalanceClient = await token.balanceOf(clientAddress) - - check endBalanceClient == (startBalanceClient + request.totalPrice) - - test "supports request subscriptions": - var receivedIds: seq[RequestId] - var receivedAsks: seq[StorageAsk] - proc onRequest(id: RequestId, ask: StorageAsk, expiry: uint64) = - receivedIds.add(id) - receivedAsks.add(ask) - - let subscription = await market.subscribeRequests(onRequest) - await market.requestStorage(request) - - check eventually receivedIds == @[request.id] and receivedAsks == @[request.ask] - await subscription.unsubscribe() - - test "supports filling of slots": - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - - test "can retrieve host that filled slot": - await market.requestStorage(request) - check (await market.getHost(request.id, slotIndex)) == none Address - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - check (await market.getHost(request.id, slotIndex)) == some accounts[0] - - test "supports freeing a slot": - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - await market.freeSlot(slotId(request.id, slotIndex)) - check (await market.getHost(request.id, slotIndex)) == none Address - - test "supports checking whether proof is required now": - check (await market.isProofRequired(slotId(request.id, slotIndex))) == false - - test "supports checking whether proof is required soon": - check (await market.willProofBeRequired(slotId(request.id, slotIndex))) == false - - test "submits proofs": - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - await advanceToNextPeriod() - await market.submitProof(slotId(request.id, slotIndex), proof) - - test "marks a proof as missing": - let slotId = slotId(request, slotIndex) - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - await waitUntilProofRequired(slotId) - let missingPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await advanceToNextPeriod() - await market.markProofAsMissing(slotId, missingPeriod) - check (await marketplace.missingProofs(slotId)) == 1 - - test "can check whether a proof can be marked as missing": - let slotId = slotId(request, slotIndex) - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - await waitUntilProofRequired(slotId) - let missingPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await advanceToNextPeriod() - check (await market.canMarkProofAsMissing(slotId, missingPeriod)) == true - - test "can check whether a proof cannot be marked as missing when the slot is free": - let slotId = slotId(request, slotIndex) - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - await waitUntilProofRequired(slotId) - - await market.freeSlot(slotId(request.id, slotIndex)) - - let missingPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await advanceToNextPeriod() - check (await market.canMarkProofAsMissing(slotId, missingPeriod)) == false - - test "can check whether a proof cannot be marked as missing before a proof is required": - let slotId = slotId(request, slotIndex) - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - - let missingPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await advanceToNextPeriod() - check (await market.canMarkProofAsMissing(slotId, missingPeriod)) == false - - test "can check whether a proof cannot be marked as missing if the proof was submitted": - let slotId = slotId(request, slotIndex) - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - await waitUntilProofRequired(slotId) - - await market.submitProof(slotId(request.id, slotIndex), proof) - - let missingPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await advanceToNextPeriod() - check (await market.canMarkProofAsMissing(slotId, missingPeriod)) == false - - test "supports slot filled subscriptions": - await market.requestStorage(request) - var receivedIds: seq[RequestId] - var receivedSlotIndices: seq[uint64] - proc onSlotFilled(id: RequestId, slotIndex: uint64) = - receivedIds.add(id) - receivedSlotIndices.add(slotIndex) - - let subscription = await market.subscribeSlotFilled(onSlotFilled) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - check eventually receivedIds == @[request.id] and receivedSlotIndices == @[ - slotIndex - ] - await subscription.unsubscribe() - - test "subscribes only to a certain slot": - var otherSlot = slotIndex - 1 - await market.requestStorage(request) - var receivedSlotIndices: seq[uint64] - proc onSlotFilled(requestId: RequestId, slotIndex: uint64) = - receivedSlotIndices.add(slotIndex) - - let subscription = - await market.subscribeSlotFilled(request.id, slotIndex, onSlotFilled) - await market.reserveSlot(request.id, otherSlot) - await market.fillSlot(request.id, otherSlot, proof, request.ask.collateralPerSlot) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - check eventually receivedSlotIndices == @[slotIndex] - await subscription.unsubscribe() - - test "supports slot freed subscriptions": - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - var receivedRequestIds: seq[RequestId] = @[] - var receivedIdxs: seq[uint64] = @[] - proc onSlotFreed(requestId: RequestId, idx: uint64) = - receivedRequestIds.add(requestId) - receivedIdxs.add(idx) - - let subscription = await market.subscribeSlotFreed(onSlotFreed) - await market.freeSlot(slotId(request.id, slotIndex)) - check eventually receivedRequestIds == @[request.id] and receivedIdxs == @[ - slotIndex - ] - await subscription.unsubscribe() - - test "supports slot reservations full subscriptions": - let account2 = ethProvider.getSigner(accounts[2]) - let account3 = ethProvider.getSigner(accounts[3]) - - await market.requestStorage(request) - - var receivedRequestIds: seq[RequestId] = @[] - var receivedIdxs: seq[uint64] = @[] - proc onSlotReservationsFull(requestId: RequestId, idx: uint64) = - receivedRequestIds.add(requestId) - receivedIdxs.add(idx) - - let subscription = - await market.subscribeSlotReservationsFull(onSlotReservationsFull) - - await market.reserveSlot(request.id, slotIndex) - switchAccount(account2) - await market.reserveSlot(request.id, slotIndex) - switchAccount(account3) - await market.reserveSlot(request.id, slotIndex) - - check eventually receivedRequestIds == @[request.id] and receivedIdxs == @[ - slotIndex - ] - await subscription.unsubscribe() - - test "support fulfillment subscriptions": - await market.requestStorage(request) - var receivedIds: seq[RequestId] - proc onFulfillment(id: RequestId) = - receivedIds.add(id) - - let subscription = await market.subscribeFulfillment(request.id, onFulfillment) - for slotIndex in 0 ..< request.ask.slots: - await market.reserveSlot(request.id, slotIndex.uint64) - await market.fillSlot( - request.id, slotIndex.uint64, proof, request.ask.collateralPerSlot - ) - check eventually receivedIds == @[request.id] - await subscription.unsubscribe() - - test "subscribes only to fulfillment of a certain request": - var otherRequest = StorageRequest.example - otherRequest.client = accounts[0] - - await market.requestStorage(request) - await market.requestStorage(otherRequest) - - var receivedIds: seq[RequestId] - proc onFulfillment(id: RequestId) = - receivedIds.add(id) - - let subscription = await market.subscribeFulfillment(request.id, onFulfillment) - - for slotIndex in 0 ..< request.ask.slots: - await market.reserveSlot(request.id, slotIndex.uint64) - await market.fillSlot( - request.id, slotIndex.uint64, proof, request.ask.collateralPerSlot - ) - for slotIndex in 0 ..< otherRequest.ask.slots: - await market.reserveSlot(otherRequest.id, slotIndex.uint64) - await market.fillSlot( - otherRequest.id, slotIndex.uint64, proof, otherRequest.ask.collateralPerSlot - ) - - check eventually receivedIds == @[request.id] - - await subscription.unsubscribe() - - test "support request cancelled subscriptions": - await market.requestStorage(request) - - var receivedIds: seq[RequestId] - proc onRequestCancelled(id: RequestId) = - receivedIds.add(id) - - let subscription = - await market.subscribeRequestCancelled(request.id, onRequestCancelled) - - await advanceToCancelledRequest(request) - await market.withdrawFunds(request.id) - check eventually receivedIds == @[request.id] - await subscription.unsubscribe() - - test "support request failed subscriptions": - await market.requestStorage(request) - - var receivedIds: seq[RequestId] - proc onRequestFailed(id: RequestId) = - receivedIds.add(id) - - let subscription = await market.subscribeRequestFailed(request.id, onRequestFailed) - - for slotIndex in 0 ..< request.ask.slots: - await market.reserveSlot(request.id, slotIndex.uint64) - await market.fillSlot( - request.id, slotIndex.uint64, proof, request.ask.collateralPerSlot - ) - for slotIndex in 0 .. request.ask.maxSlotLoss: - let slotId = request.slotId(slotIndex.uint64) - while true: - let slotState = await marketplace.slotState(slotId) - if slotState == SlotState.Repair or slotState == SlotState.Failed: - break - await waitUntilProofRequired(slotId) - let missingPeriod = - periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - await advanceToNextPeriod() - discard await marketplace.markProofAsMissing(slotId, missingPeriod).confirm(1) - check eventually receivedIds == @[request.id] - await subscription.unsubscribe() - - test "subscribes only to a certain request cancellation": - var otherRequest = request - otherRequest.nonce = Nonce.example - await market.requestStorage(request) - await market.requestStorage(otherRequest) - - var receivedIds: seq[RequestId] - proc onRequestCancelled(requestId: RequestId) = - receivedIds.add(requestId) - - let subscription = - await market.subscribeRequestCancelled(request.id, onRequestCancelled) - await advanceToCancelledRequest(otherRequest) # shares expiry with otherRequest - await market.withdrawFunds(otherRequest.id) - await market.withdrawFunds(request.id) - check eventually receivedIds == @[request.id] - await subscription.unsubscribe() - - test "supports proof submission subscriptions": - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - await advanceToNextPeriod() - var receivedIds: seq[SlotId] - proc onProofSubmission(id: SlotId) = - receivedIds.add(id) - - let subscription = await market.subscribeProofSubmission(onProofSubmission) - await market.submitProof(slotId(request.id, slotIndex), proof) - check eventually receivedIds == @[slotId(request.id, slotIndex)] - await subscription.unsubscribe() - - test "request is none when unknown": - check isNone await market.getRequest(request.id) - - test "can retrieve active requests": - await market.requestStorage(request) - var request2 = StorageRequest.example - request2.client = accounts[0] - await market.requestStorage(request2) - check (await market.myRequests()) == @[request.id, request2.id] - - test "retrieves correct request state when request is unknown": - check (await market.requestState(request.id)) == none RequestState - - test "can retrieve request state": - await market.requestStorage(request) - for slotIndex in 0 ..< request.ask.slots: - await market.reserveSlot(request.id, slotIndex.uint64) - await market.fillSlot( - request.id, slotIndex.uint64, proof, request.ask.collateralPerSlot - ) - check (await market.requestState(request.id)) == some RequestState.Started - - test "can retrieve active slots": - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex - 1) - await market.fillSlot( - request.id, slotIndex - 1, proof, request.ask.collateralPerSlot - ) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - let slotId1 = request.slotId(slotIndex - 1) - let slotId2 = request.slotId(slotIndex) - check (await market.mySlots()) == @[slotId1, slotId2] - - test "returns none when slot is empty": - await market.requestStorage(request) - let slotId = request.slotId(slotIndex) - check (await market.getActiveSlot(slotId)) == none Slot - - test "can retrieve request details from slot id": - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - let slotId = request.slotId(slotIndex) - let expected = Slot(request: request, slotIndex: slotIndex) - check (await market.getActiveSlot(slotId)) == some expected - - test "retrieves correct slot state when request is unknown": - let slotId = request.slotId(slotIndex) - check (await market.slotState(slotId)) == SlotState.Free - - test "retrieves correct slot state once filled": - await market.requestStorage(request) - await market.reserveSlot(request.id, slotIndex) - await market.fillSlot(request.id, slotIndex, proof, request.ask.collateralPerSlot) - let slotId = request.slotId(slotIndex) - check (await market.slotState(slotId)) == SlotState.Filled - - test "can query past StorageRequested events": - var request1 = StorageRequest.example - var request2 = StorageRequest.example - request1.client = accounts[0] - request2.client = accounts[0] - await market.requestStorage(request) - await market.requestStorage(request1) - await market.requestStorage(request2) - - # `market.requestStorage` executes an `approve` tx before the - # `requestStorage` tx, so that's two PoA blocks per `requestStorage` call (6 - # blocks for 3 calls). We don't need to check the `approve` for the first - # `requestStorage` call, so we only need to check 5 "blocks ago". "blocks - # ago". - - proc getsPastRequest(): Future[bool] {.async.} = - let reqs = await market.queryPastStorageRequestedEvents(blocksAgo = 5) - reqs.mapIt(it.requestId) == @[request.id, request1.id, request2.id] - - check eventually await getsPastRequest() - - test "can query past SlotFilled events": - await market.requestStorage(request) - await market.reserveSlot(request.id, 0.uint64) - await market.reserveSlot(request.id, 1.uint64) - await market.reserveSlot(request.id, 2.uint64) - await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - await market.fillSlot(request.id, 1.uint64, proof, request.ask.collateralPerSlot) - await market.fillSlot(request.id, 2.uint64, proof, request.ask.collateralPerSlot) - - # `market.fill` executes an `approve` tx before the `fillSlot` tx, so that's - # two PoA blocks per `fillSlot` call (6 blocks for 3 calls). We don't need - # to check the `approve` for the first `fillSlot` call, so we only need to - # check 5 "blocks ago". - let events = await market.queryPastSlotFilledEvents(blocksAgo = 5) - check events == - @[ - SlotFilled(requestId: request.id, slotIndex: 0), - SlotFilled(requestId: request.id, slotIndex: 1), - SlotFilled(requestId: request.id, slotIndex: 2), - ] - - test "can query past SlotFilled events since given timestamp": - await market.requestStorage(request) - await market.reserveSlot(request.id, 0.uint64) - await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - - # The SlotFilled event will be included in the same block as - # the fillSlot transaction. If we want to ignore the SlotFilled event - # for this first slot, we need to jump to the next block and use the - # timestamp of that block as our "fromTime" parameter to the - # queryPastSlotFilledEvents function. - await ethProvider.advanceTime(10.u256) - - let (_, fromTime) = await ethProvider.blockNumberAndTimestamp(BlockTag.latest) - - await ethProvider.advanceTime(1.u256) - - await market.reserveSlot(request.id, 1.uint64) - await market.reserveSlot(request.id, 2.uint64) - await market.fillSlot(request.id, 1.uint64, proof, request.ask.collateralPerSlot) - await market.fillSlot(request.id, 2.uint64, proof, request.ask.collateralPerSlot) - - let events = await market.queryPastSlotFilledEvents( - fromTime = fromTime.truncate(SecondsSince1970) - ) - - check events == - @[ - SlotFilled(requestId: request.id, slotIndex: 1), - SlotFilled(requestId: request.id, slotIndex: 2), - ] - - test "queryPastSlotFilledEvents returns empty sequence of events when " & - "no SlotFilled events have occurred since given timestamp": - await market.requestStorage(request) - await market.reserveSlot(request.id, 0.uint64) - await market.reserveSlot(request.id, 1.uint64) - await market.reserveSlot(request.id, 2.uint64) - await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - await market.fillSlot(request.id, 1.uint64, proof, request.ask.collateralPerSlot) - await market.fillSlot(request.id, 2.uint64, proof, request.ask.collateralPerSlot) - - await ethProvider.advanceTime(10.u256) - - let (_, fromTime) = await ethProvider.blockNumberAndTimestamp(BlockTag.latest) - - let events = await market.queryPastSlotFilledEvents( - fromTime = fromTime.truncate(SecondsSince1970) - ) - - check events.len == 0 - - test "past event query can specify negative `blocksAgo` parameter": - await market.requestStorage(request) - - check eventually ( - (await market.queryPastStorageRequestedEvents(blocksAgo = -2)) == - (await market.queryPastStorageRequestedEvents(blocksAgo = 2)) - ) - - test "pays rewards and returns collateral to host": - await market.requestStorage(request) - - let address = await host.getAddress() - switchAccount(host) - await market.reserveSlot(request.id, 0.uint64) - await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - let filledAt = await ethProvider.blockTime(BlockTag.latest) - - for slotIndex in 1 ..< request.ask.slots: - await market.reserveSlot(request.id, slotIndex.uint64) - await market.fillSlot( - request.id, slotIndex.uint64, proof, request.ask.collateralPerSlot - ) - - let requestEnd = await market.getRequestEnd(request.id) - await ethProvider.advanceTimeTo(requestEnd.u256 + 1) - - let startBalance = await token.balanceOf(address) - await market.freeSlot(request.slotId(0.uint64)) - let endBalance = await token.balanceOf(address) - - let expectedPayout = request.expectedPayout(filledAt, requestEnd.u256) - check endBalance == (startBalance + expectedPayout + request.ask.collateralPerSlot) - - test "pays rewards to reward recipient, collateral to host": - market = OnChainMarket.new(marketplace, hostRewardRecipient.some) - let hostAddress = await host.getAddress() - - await market.requestStorage(request) - - switchAccount(host) - await market.reserveSlot(request.id, 0.uint64) - await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - let filledAt = await ethProvider.blockTime(BlockTag.latest) - - for slotIndex in 1 ..< request.ask.slots: - await market.reserveSlot(request.id, slotIndex.uint64) - await market.fillSlot( - request.id, slotIndex.uint64, proof, request.ask.collateralPerSlot - ) - - let requestEnd = await market.getRequestEnd(request.id) - await ethProvider.advanceTimeTo(requestEnd.u256 + 1) - - let startBalanceHost = await token.balanceOf(hostAddress) - let startBalanceReward = await token.balanceOf(hostRewardRecipient) - - await market.freeSlot(request.slotId(0.uint64)) - - let endBalanceHost = await token.balanceOf(hostAddress) - let endBalanceReward = await token.balanceOf(hostRewardRecipient) - - let expectedPayout = request.expectedPayout(filledAt, requestEnd.u256) - check endBalanceHost == (startBalanceHost + request.ask.collateralPerSlot) - check endBalanceReward == (startBalanceReward + expectedPayout) - - test "returns the collateral when the slot is not being repaired": - await market.requestStorage(request) - await market.reserveSlot(request.id, 0.uint64) - await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - - let slotId = request.slotId(0.uint64) - without collateral =? await market.slotCollateral(request.id, 0.uint64), error: - fail() - - check collateral == request.ask.collateralPerSlot - - test "calculates correctly the collateral when the slot is being repaired": - # Ensure that the config is loaded and repairRewardPercentage is available - discard await market.repairRewardPercentage() - - await market.requestStorage(request) - await market.reserveSlot(request.id, 0.uint64) - await market.fillSlot(request.id, 0.uint64, proof, request.ask.collateralPerSlot) - await market.freeSlot(slotId(request.id, 0.uint64)) - - let slotId = request.slotId(0.uint64) - - without collateral =? await market.slotCollateral(request.id, 0.uint64), error: - fail() - - # slotCollateral - # repairRewardPercentage = 10 - # expected collateral = slotCollateral - slotCollateral * 0.1 - check collateral == - request.ask.collateralPerSlot - (request.ask.collateralPerSlot * 10).div(100.u256) - - test "the request is added to cache after the first access": - await market.requestStorage(request) - - check market.requestCache.contains($request.id) == false - discard await market.getRequest(request.id) - - check market.requestCache.contains($request.id) == true - let cacheValue = market.requestCache[$request.id] - check cacheValue == request diff --git a/tests/contracts/testProvider.nim b/tests/contracts/testProvider.nim deleted file mode 100644 index 0414f7c6..00000000 --- a/tests/contracts/testProvider.nim +++ /dev/null @@ -1,161 +0,0 @@ -import pkg/chronos -import codex/contracts -import ../asynctest -import ../ethertest -import ./time -import ./helpers/mockprovider - -# to see supportive information in the test output -# use `-d:"chronicles_enabled_topics:testProvider:DEBUG` option -# when compiling the test file -logScope: - topics = "testProvider" - -suite "Provider (Mock)": - test "blockNumberForEpoch returns the earliest block when its timestamp " & - "is greater than the given epoch time and the earliest block is not " & - "block number 0 (genesis block)": - let mockProvider = newMockProvider( - numberOfBlocks = 10, - earliestBlockNumber = 1, - earliestBlockTimestamp = 10, - timeIntervalBetweenBlocks = 10, - ) - - let (earliestBlockNumber, earliestTimestamp) = - await mockProvider.blockNumberAndTimestamp(BlockTag.earliest) - - let epochTime = earliestTimestamp - 1 - - let actual = - await mockProvider.blockNumberForEpoch(epochTime.truncate(SecondsSince1970)) - - check actual == earliestBlockNumber - - test "blockNumberForEpoch returns the earliest block when its timestamp " & - "is equal to the given epoch time": - let mockProvider = newMockProvider( - numberOfBlocks = 10, - earliestBlockNumber = 0, - earliestBlockTimestamp = 10, - timeIntervalBetweenBlocks = 10, - ) - - let (earliestBlockNumber, earliestTimestamp) = - await mockProvider.blockNumberAndTimestamp(BlockTag.earliest) - - let epochTime = earliestTimestamp - - let actual = - await mockProvider.blockNumberForEpoch(epochTime.truncate(SecondsSince1970)) - - check earliestBlockNumber == 0.u256 - check actual == earliestBlockNumber - - test "blockNumberForEpoch returns the latest block when its timestamp " & - "is equal to the given epoch time": - let mockProvider = newMockProvider( - numberOfBlocks = 10, - earliestBlockNumber = 0, - earliestBlockTimestamp = 10, - timeIntervalBetweenBlocks = 10, - ) - - let (latestBlockNumber, latestTimestamp) = - await mockProvider.blockNumberAndTimestamp(BlockTag.latest) - - let epochTime = latestTimestamp - - let actual = - await mockProvider.blockNumberForEpoch(epochTime.truncate(SecondsSince1970)) - - check actual == latestBlockNumber - -ethersuite "Provider": - proc mineNBlocks(provider: JsonRpcProvider, n: int) {.async.} = - for _ in 0 ..< n: - discard await provider.send("evm_mine") - - test "blockNumberForEpoch finds closest blockNumber for given epoch time": - proc createBlockHistory( - n: int, blockTime: int - ): Future[seq[(UInt256, UInt256)]] {.async.} = - var blocks: seq[(UInt256, UInt256)] = @[] - for _ in 0 ..< n: - await ethProvider.advanceTime(blockTime.u256) - let (blockNumber, blockTimestamp) = - await ethProvider.blockNumberAndTimestamp(BlockTag.latest) - # collect blocknumbers and timestamps - blocks.add((blockNumber, blockTimestamp)) - blocks - - proc printBlockNumbersAndTimestamps(blocks: seq[(UInt256, UInt256)]) = - for (blockNumber, blockTimestamp) in blocks: - debug "Block", blockNumber = blockNumber, timestamp = blockTimestamp - - type Expectations = tuple[epochTime: UInt256, expectedBlockNumber: UInt256] - - # We want to test that timestamps at the block boundaries, in the middle, - # and towards lower and upper part of the range are correctly mapped to - # the closest block number. - # For example: assume we have the following two blocks with - # the corresponding block numbers and timestamps: - # block1: (291, 1728436100) - # block2: (292, 1728436110) - # To test that binary search correctly finds the closest block number, - # we will test the following timestamps: - # 1728436100 => 291 - # 1728436104 => 291 - # 1728436105 => 292 - # 1728436106 => 292 - # 1728436110 => 292 - proc generateExpectations(blocks: seq[(UInt256, UInt256)]): seq[Expectations] = - var expectations: seq[Expectations] = @[] - for i in 0 ..< blocks.len - 1: - let (startNumber, startTimestamp) = blocks[i] - let (endNumber, endTimestamp) = blocks[i + 1] - let middleTimestamp = (startTimestamp + endTimestamp) div 2 - let lowerExpectation = (middleTimestamp - 1, startNumber) - expectations.add((startTimestamp, startNumber)) - expectations.add(lowerExpectation) - if middleTimestamp.truncate(int64) - startTimestamp.truncate(int64) < - endTimestamp.truncate(int64) - middleTimestamp.truncate(int64): - expectations.add((middleTimestamp, startNumber)) - else: - expectations.add((middleTimestamp, endNumber)) - let higherExpectation = (middleTimestamp + 1, endNumber) - expectations.add(higherExpectation) - if i == blocks.len - 2: - expectations.add((endTimestamp, endNumber)) - expectations - - proc printExpectations(expectations: seq[Expectations]) = - debug "Expectations", numberOfExpectations = expectations.len - for (epochTime, expectedBlockNumber) in expectations: - debug "Expectation", - epochTime = epochTime, expectedBlockNumber = expectedBlockNumber - - # mark the beginning of the history for our test - await ethProvider.mineNBlocks(1) - - # set average block time - 10s - we use larger block time - # then expected in Linea for more precise testing of the binary search - let averageBlockTime = 10 - - # create a history of N blocks - let N = 10 - let blocks = await createBlockHistory(N, averageBlockTime) - - printBlockNumbersAndTimestamps(blocks) - - # generate expectations for block numbers - let expectations = generateExpectations(blocks) - printExpectations(expectations) - - # validate expectations - for (epochTime, expectedBlockNumber) in expectations: - debug "Validating", - epochTime = epochTime, expectedBlockNumber = expectedBlockNumber - let actualBlockNumber = - await ethProvider.blockNumberForEpoch(epochTime.truncate(SecondsSince1970)) - check actualBlockNumber == expectedBlockNumber diff --git a/tests/contracts/time.nim b/tests/contracts/time.nim deleted file mode 100644 index 3d038fc2..00000000 --- a/tests/contracts/time.nim +++ /dev/null @@ -1,18 +0,0 @@ -import pkg/ethers -import pkg/serde/json - -proc blockTime*(provider: Provider, tag: BlockTag): Future[UInt256] {.async.} = - return (!await provider.getBlock(tag)).timestamp - -proc currentTime*(provider: Provider): Future[UInt256] {.async.} = - return await provider.blockTime(BlockTag.pending) - -proc advanceTime*(provider: JsonRpcProvider, seconds: UInt256) {.async.} = - discard await provider.send("evm_increaseTime", @[%("0x" & seconds.toHex)]) - discard await provider.send("evm_mine") - -proc advanceTimeTo*(provider: JsonRpcProvider, timestamp: UInt256) {.async.} = - if (await provider.currentTime()) != timestamp: - discard - await provider.send("evm_setNextBlockTimestamp", @[%("0x" & timestamp.toHex)]) - discard await provider.send("evm_mine") diff --git a/tests/examples.nim b/tests/examples.nim index 2b7e96d6..4d413c3c 100644 --- a/tests/examples.nim +++ b/tests/examples.nim @@ -3,10 +3,7 @@ import std/sequtils import std/times import std/typetraits -import pkg/codex/contracts/requests import pkg/codex/rng -import pkg/codex/contracts/proofs -import pkg/codex/sales/slotqueue import pkg/codex/stores import pkg/codex/units @@ -44,50 +41,6 @@ proc example*[T: distinct](_: type T): T = type baseType = T.distinctBase T(baseType.example) -proc example*(_: type StorageRequest): StorageRequest = - StorageRequest( - client: Address.example, - ask: StorageAsk( - slots: 4, - slotSize: (1 * 1024 * 1024 * 1024).uint64, # 1 Gigabyte - duration: (10 * 60 * 60).uint64, # 10 hours - collateralPerByte: 1.u256, - proofProbability: 4.u256, # require a proof roughly once every 4 periods - pricePerBytePerSecond: 1.u256, - maxSlotLoss: 2, # 2 slots can be freed without data considered to be lost - ), - content: StorageContent( - cid: Cid.init("zb2rhheVmk3bLks5MgzTqyznLu1zqGH5jrfTA1eAZXrjx7Vob").tryGet, - merkleRoot: array[32, byte].example, - ), - expiry: (60 * 60).uint64, # 1 hour , - nonce: Nonce.example, - ) - -proc example*(_: type Slot): Slot = - let request = StorageRequest.example - let slotIndex = rand(request.ask.slots.int).uint64 - Slot(request: request, slotIndex: slotIndex) - -proc example*(_: type SlotQueueItem): SlotQueueItem = - let request = StorageRequest.example - let slot = Slot.example - SlotQueueItem.init( - request, slot.slotIndex.uint16, collateral = request.ask.collateralPerSlot - ) - -proc example(_: type G1Point): G1Point = - G1Point(x: UInt256.example, y: UInt256.example) - -proc example(_: type G2Point): G2Point = - G2Point( - x: Fp2Element(real: UInt256.example, imag: UInt256.example), - y: Fp2Element(real: UInt256.example, imag: UInt256.example), - ) - -proc example*(_: type Groth16Proof): Groth16Proof = - Groth16Proof(a: G1Point.example, b: G2Point.example, c: G1Point.example) - proc example*(_: type RandomChunker, blocks: int): Future[seq[byte]] {.async.} = let rng = Rng.instance() let chunker = RandomChunker.new( diff --git a/tests/integration/1_minute/testblockexpiration.nim b/tests/integration/1_minute/testblockexpiration.nim index ee558c90..be66c388 100644 --- a/tests/integration/1_minute/testblockexpiration.nim +++ b/tests/integration/1_minute/testblockexpiration.nim @@ -12,8 +12,7 @@ multinodesuite "Node block expiration tests": clients: CodexConfigs .init(nodes = 1) .withBlockTtl(0, 10) - .withBlockMaintenanceInterval(0, 1).some, - providers: CodexConfigs.none, + .withBlockMaintenanceInterval(0, 1).some ): let client = clients()[0] let clientApi = client.client @@ -33,8 +32,7 @@ multinodesuite "Node block expiration tests": clients: CodexConfigs .init(nodes = 1) .withBlockTtl(0, 1) - .withBlockMaintenanceInterval(0, 1).some, - providers: CodexConfigs.none, + .withBlockMaintenanceInterval(0, 1).some ): let client = clients()[0] let clientApi = client.client diff --git a/tests/integration/1_minute/testcli.nim b/tests/integration/1_minute/testcli.nim deleted file mode 100644 index 778608b8..00000000 --- a/tests/integration/1_minute/testcli.nim +++ /dev/null @@ -1,60 +0,0 @@ -import std/tempfiles -import codex/conf -import codex/utils/fileutils -import ../../asynctest -import ../../checktest -import ../codexprocess -import ../nodeprocess -import ../../examples - -asyncchecksuite "Command line interface": - let key = "4242424242424242424242424242424242424242424242424242424242424242" - - proc startCodex(args: seq[string]): Future[CodexProcess] {.async.} = - return await CodexProcess.startNode(args, false, "cli-test-node") - - test "complains when persistence is enabled without ethereum account": - let node = await startCodex(@["persistence"]) - await node.waitUntilOutput("Persistence enabled, but no Ethereum account was set") - await node.stop() - - test "complains when ethereum private key file has wrong permissions": - let unsafeKeyFile = genTempPath("", "") - discard unsafeKeyFile.writeFile(key, 0o666) - let node = await startCodex(@["persistence", "--eth-private-key=" & unsafeKeyFile]) - await node.waitUntilOutput( - "Ethereum private key file does not have safe file permissions" - ) - await node.stop() - discard removeFile(unsafeKeyFile) - - let - marketplaceArg = "--marketplace-address=" & $EthAddress.example - expectedDownloadInstruction = - "Proving circuit files are not found. Please run the following to download them:" - - test "suggests downloading of circuit files when persistence is enabled without accessible r1cs file": - let node = await startCodex(@["persistence", "prover", marketplaceArg]) - await node.waitUntilOutput(expectedDownloadInstruction) - await node.stop() - - test "suggests downloading of circuit files when persistence is enabled without accessible wasm file": - let node = await startCodex( - @[ - "persistence", "prover", marketplaceArg, - "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs", - ] - ) - await node.waitUntilOutput(expectedDownloadInstruction) - await node.stop() - - test "suggests downloading of circuit files when persistence is enabled without accessible zkey file": - let node = await startCodex( - @[ - "persistence", "prover", marketplaceArg, - "--circom-r1cs=tests/circuits/fixtures/proof_main.r1cs", - "--circom-wasm=tests/circuits/fixtures/proof_main.wasm", - ] - ) - await node.waitUntilOutput(expectedDownloadInstruction) - await node.stop() diff --git a/tests/integration/1_minute/testecbug.nim b/tests/integration/1_minute/testecbug.nim deleted file mode 100644 index a5bfa832..00000000 --- a/tests/integration/1_minute/testecbug.nim +++ /dev/null @@ -1,59 +0,0 @@ -from pkg/libp2p import Cid, init -import ../../examples -import ../marketplacesuite -import ../nodeconfigs -import ../hardhatconfig - -marketplacesuite( - name = "Bug #821 - node crashes during erasure coding", stopOnRequestFail = true -): - test "should be able to create storage request and download dataset", - NodeConfigs( - clients: CodexConfigs - .init(nodes = 1) - # .debug() # uncomment to enable console log output.debug() - .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node", "erasure", "marketplace").some, - providers: CodexConfigs.init(nodes = 0).some, - ): - let - pricePerBytePerSecond = 1.u256 - duration = 20.periods - collateralPerByte = 1.u256 - expiry = 10.periods - data = await RandomChunker.example(blocks = 8) - client = clients()[0] - clientApi = client.client - - let cid = (await clientApi.upload(data)).get - - var requestId = none RequestId - proc onStorageRequested(eventResult: ?!StorageRequested) = - assert not eventResult.isErr - requestId = some (!eventResult).requestId - - let subscription = await marketplace.subscribe(StorageRequested, onStorageRequested) - - # client requests storage but requires multiple slots to host the content - let id = await clientApi.requestStorage( - cid, - duration = duration, - pricePerBytePerSecond = pricePerBytePerSecond, - expiry = expiry, - collateralPerByte = collateralPerByte, - nodes = 3, - tolerance = 1, - ) - - check eventually(requestId.isSome, timeout = expiry.int * 1000) - - let - request = await marketplace.getRequest(requestId.get) - cidFromRequest = request.content.cid - downloaded = await clientApi.downloadBytes(cidFromRequest, local = true) - - check downloaded.isOk - check downloaded.get.toHex == data.toHex - - await subscription.unsubscribe() diff --git a/tests/integration/1_minute/testpurchasing.nim.ignore b/tests/integration/1_minute/testpurchasing.nim.ignore deleted file mode 100644 index 7a394c0c..00000000 --- a/tests/integration/1_minute/testpurchasing.nim.ignore +++ /dev/null @@ -1,140 +0,0 @@ -import std/options -import std/httpclient -import pkg/codex/rng -import ../twonodes -import ../../contracts/time -import ../../examples - -twonodessuite "Purchasing": - test "node handles storage request", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let id1 = ( - await client1.requestStorage( - cid, - duration = 100.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - expiry = 10.uint64, - collateralPerByte = 1.u256, - ) - ).get - let id2 = ( - await client1.requestStorage( - cid, - duration = 400.uint64, - pricePerBytePerSecond = 2.u256, - proofProbability = 6.u256, - expiry = 10.uint64, - collateralPerByte = 2.u256, - ) - ).get - check id1 != id2 - - test "node retrieves purchase status", twoNodesConfig: - # get one contiguous chunk - let rng = rng.Rng.instance() - let chunker = RandomChunker.new( - rng, size = DefaultBlockSize * 2, chunkSize = DefaultBlockSize * 2 - ) - let data = await chunker.getBytes() - let cid = (await client1.upload(byteutils.toHex(data))).get - let id = ( - await client1.requestStorage( - cid, - duration = 100.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - expiry = 30.uint64, - collateralPerByte = 1.u256, - nodes = 3, - tolerance = 1, - ) - ).get - - let request = (await client1.getPurchase(id)).get.request.get - - check request.content.cid.data.buffer.len > 0 - check request.ask.duration == 100.uint64 - check request.ask.pricePerBytePerSecond == 1.u256 - check request.ask.proofProbability == 3.u256 - check request.expiry == 30.uint64 - check request.ask.collateralPerByte == 1.u256 - check request.ask.slots == 3'u64 - check request.ask.maxSlotLoss == 1'u64 - - # TODO: We currently do not support encoding single chunks - # test "node retrieves purchase status with 1 chunk", twoNodesConfig: - # let cid = client1.upload("some file contents").get - # let id = client1.requestStorage( - # cid, duration=1.u256, pricePerBytePerSecond=1.u256, - # proofProbability=3.u256, expiry=30, collateralPerByte=1.u256, - # nodes=2, tolerance=1).get - # let request = client1.getPurchase(id).get.request.get - # check request.ask.duration == 1.u256 - # check request.ask.pricePerBytePerSecond == 1.u256 - # check request.ask.proofProbability == 3.u256 - # check request.expiry == 30 - # check request.ask.collateralPerByte == 1.u256 - # check request.ask.slots == 3'u64 - # check request.ask.maxSlotLoss == 1'u64 - - test "node remembers purchase status after restart", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let id = ( - await client1.requestStorage( - cid, - duration = 10 * 60.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - expiry = 5 * 60.uint64, - collateralPerByte = 1.u256, - nodes = 3.uint, - tolerance = 1.uint, - ) - ).get - check eventually( - await client1.purchaseStateIs(id, "submitted"), timeout = 3 * 60 * 1000 - ) - - await node1.restart() - - check eventually( - await client1.purchaseStateIs(id, "submitted"), timeout = 3 * 60 * 1000 - ) - let request = (await client1.getPurchase(id)).get.request.get - check request.ask.duration == (10 * 60).uint64 - check request.ask.pricePerBytePerSecond == 1.u256 - check request.ask.proofProbability == 3.u256 - check request.expiry == (5 * 60).uint64 - check request.ask.collateralPerByte == 1.u256 - check request.ask.slots == 3'u64 - check request.ask.maxSlotLoss == 1'u64 - - test "node requires expiry and its value to be in future", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - - let responseMissing = await client1.requestStorageRaw( - cid, - duration = 1.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - collateralPerByte = 1.u256, - ) - check responseMissing.status == 422 - check (await responseMissing.body) == - "Expiry must be greater than zero and less than the request's duration" - - let responseBefore = await client1.requestStorageRaw( - cid, - duration = 10.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - collateralPerByte = 1.u256, - expiry = 10.uint64, - ) - check responseBefore.status == 422 - check "Expiry must be greater than zero and less than the request's duration" in - (await responseBefore.body) diff --git a/tests/integration/30_minutes/.gitkeep b/tests/integration/30_minutes/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/30_minutes/testmarketplace.nim.ignore b/tests/integration/30_minutes/testmarketplace.nim.ignore deleted file mode 100644 index b04626c4..00000000 --- a/tests/integration/30_minutes/testmarketplace.nim.ignore +++ /dev/null @@ -1,397 +0,0 @@ -import std/times -import std/httpclient -import ../../examples -import ../../contracts/time -import ../../contracts/deployment -import ./../marketplacesuite -import ../twonodes -import ../nodeconfigs - -marketplacesuite(name = "Marketplace", stopOnRequestFail = true): - let marketplaceConfig = NodeConfigs( - clients: CodexConfigs.init(nodes = 1).some, - providers: CodexConfigs.init(nodes = 1).some, - ) - - var host: CodexClient - var hostAccount: Address - var client: CodexClient - var clientAccount: Address - - const minPricePerBytePerSecond = 1.u256 - const collateralPerByte = 1.u256 - const blocks = 8 - const ecNodes = 3 - const ecTolerance = 1 - - setup: - host = providers()[0].client - hostAccount = providers()[0].ethAccount - client = clients()[0].client - clientAccount = clients()[0].ethAccount - - # Our Hardhat configuration does use automine, which means that time tracked by `ethProvider.currentTime()` is not - # advanced until blocks are mined and that happens only when transaction is submitted. - # As we use in tests ethProvider.currentTime() which uses block timestamp this can lead to synchronization issues. - await ethProvider.advanceTime(1.u256) - - test "nodes negotiate contracts on the marketplace", marketplaceConfig: - let size = 0xFFFFFF.uint64 - let data = await RandomChunker.example(blocks = blocks) - # host makes storage available - let availability = ( - await host.postAvailability( - totalSize = size, - duration = 20 * 60.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size.u256 * minPricePerBytePerSecond, - ) - ).get - - # client requests storage - let cid = (await client.upload(data)).get - let id = await client.requestStorage( - cid, - duration = 20 * 60.uint64, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 3.u256, - expiry = 10 * 60.uint64, - collateralPerByte = collateralPerByte, - nodes = ecNodes, - tolerance = ecTolerance, - ) - - discard await waitForRequestToStart() - - let purchase = (await client.getPurchase(id)).get - check purchase.error == none string - let availabilities = (await host.getAvailabilities()).get - check availabilities.len == 1 - let newSize = availabilities[0].freeSize - check newSize > 0 and newSize < size - - let reservations = (await host.getAvailabilityReservations(availability.id)).get - check reservations.len == 3 - check reservations[0].requestId == purchase.requestId - - test "node slots gets paid out and rest of tokens are returned to client", - marketplaceConfig: - let size = 0xFFFFFF.uint64 - let data = await RandomChunker.example(blocks = blocks) - let marketplace = Marketplace.new(Marketplace.address, ethProvider.getSigner()) - let tokenAddress = await marketplace.token() - let token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) - let duration = 20 * 60.uint64 - - # host makes storage available - let startBalanceHost = await token.balanceOf(hostAccount) - discard ( - await host.postAvailability( - totalSize = size, - duration = 20 * 60.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size.u256 * minPricePerBytePerSecond, - ) - ).get - - # client requests storage - let cid = (await client.upload(data)).get - let id = await client.requestStorage( - cid, - duration = duration, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 3.u256, - expiry = 10 * 60.uint64, - collateralPerByte = collateralPerByte, - nodes = ecNodes, - tolerance = ecTolerance, - ) - - discard await waitForRequestToStart() - - let purchase = (await client.getPurchase(id)).get - check purchase.error == none string - - let clientBalanceBeforeFinished = await token.balanceOf(clientAccount) - - # Proving mechanism uses blockchain clock to do proving/collect/cleanup round - # hence we must use `advanceTime` over `sleepAsync` as Hardhat does mine new blocks - # only with new transaction - await ethProvider.advanceTime(duration.u256) - - # Checking that the hosting node received reward for at least the time between - let slotSize = slotSize(blocks, ecNodes, ecTolerance) - let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize - check eventually (await token.balanceOf(hostAccount)) - startBalanceHost >= - (duration - 5 * 60).u256 * pricePerSlotPerSecond * ecNodes.u256 - - # Checking that client node receives some funds back that were not used for the host nodes - check eventually( - (await token.balanceOf(clientAccount)) - clientBalanceBeforeFinished > 0, - timeout = 10 * 1000, # give client a bit of time to withdraw its funds - ) - - test "SP are able to process slots after workers were busy with other slots and ignored them", - NodeConfigs( - clients: CodexConfigs.init(nodes = 1) - # .debug() - .some, - providers: CodexConfigs.init(nodes = 2) - # .debug() - # .withLogFile() - # .withLogTopics("marketplace", "sales", "statemachine","slotqueue", "reservations") - .some, - ): - let client0 = clients()[0] - let provider0 = providers()[0] - let provider1 = providers()[1] - let duration = 20 * 60.uint64 - - let data = await RandomChunker.example(blocks = blocks) - let slotSize = slotSize(blocks, ecNodes, ecTolerance) - - # We create an avavilability allowing the first SP to host the 3 slots. - # So the second SP will not have any availability so it will just process - # the slots and ignore them. - discard await provider0.client.postAvailability( - totalSize = 3 * slotSize.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = 3 * slotSize * minPricePerBytePerSecond, - ) - - let cid = (await client0.client.upload(data)).get - - let purchaseId = await client0.client.requestStorage( - cid, - duration = duration, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 1.u256, - expiry = 10 * 60.uint64, - collateralPerByte = collateralPerByte, - nodes = ecNodes, - tolerance = ecTolerance, - ) - - let requestId = (await client0.client.requestId(purchaseId)).get - - # We wait that the 3 slots are filled by the first SP - discard await waitForRequestToStart() - - # Here we create the same availability as previously but for the second SP. - # Meaning that, after ignoring all the slots for the first request, the second SP will process - # and host the slots for the second request. - discard await provider1.client.postAvailability( - totalSize = 3 * slotSize.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = 3 * slotSize * collateralPerByte, - ) - - let purchaseId2 = await client0.client.requestStorage( - cid, - duration = duration, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 3.u256, - expiry = 10 * 60.uint64, - collateralPerByte = collateralPerByte, - nodes = ecNodes, - tolerance = ecTolerance, - ) - let requestId2 = (await client0.client.requestId(purchaseId2)).get - - # Wait that the slots of the second request are filled - discard await waitForRequestToStart() - - # Double check, verify that our second SP hosts the 3 slots - check ((await provider1.client.getSlots()).get).len == 3 - -marketplacesuite(name = "Marketplace payouts", stopOnRequestFail = true): - const minPricePerBytePerSecond = 1.u256 - const collateralPerByte = 1.u256 - const blocks = 8 - const ecNodes = 3 - const ecTolerance = 1 - - test "expired request partially pays out for stored time", - NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: HardhatConfig.none, - clients: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output.debug() - # .withLogFile() - # # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "erasure") - .some, - providers: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() - # # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics( - # "node", "marketplace", "sales", "reservations", "node", "statemachine" - # ) - .some, - ): - let duration = 20.periods - let expiry = 10.periods - let data = await RandomChunker.example(blocks = blocks) - let client = clients()[0] - let provider = providers()[0] - let clientApi = client.client - let providerApi = provider.client - let startBalanceProvider = await token.balanceOf(provider.ethAccount) - let startBalanceClient = await token.balanceOf(client.ethAccount) - - # provider makes storage available - let datasetSize = datasetSize(blocks, ecNodes, ecTolerance) - let totalAvailabilitySize = (datasetSize div 2).truncate(uint64) - discard await providerApi.postAvailability( - # make availability size small enough that we can't fill all the slots, - # thus causing a cancellation - totalSize = totalAvailabilitySize, - duration = duration.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = collateralPerByte * totalAvailabilitySize.u256, - ) - - let cid = (await clientApi.upload(data)).get - - var slotIdxFilled = none uint64 - proc onSlotFilled(eventResult: ?!SlotFilled) = - assert not eventResult.isErr - slotIdxFilled = some (!eventResult).slotIndex - - let slotFilledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - - var requestCancelledEvent = newAsyncEvent() - proc onRequestCancelled(eventResult: ?!RequestCancelled) = - assert not eventResult.isErr - requestCancelledEvent.fire() - - let requestCancelledSubscription = - await marketplace.subscribe(RequestCancelled, onRequestCancelled) - - # client requests storage but requires multiple slots to host the content - let id = await clientApi.requestStorage( - cid, - duration = duration, - pricePerBytePerSecond = minPricePerBytePerSecond, - expiry = expiry, - collateralPerByte = collateralPerByte, - nodes = ecNodes, - tolerance = ecTolerance, - ) - - # wait until one slot is filled - check eventually(slotIdxFilled.isSome, timeout = expiry.int * 1000) - let slotId = slotId(!(await clientApi.requestId(id)), !slotIdxFilled) - - # wait until sale is cancelled - await ethProvider.advanceTime(expiry.u256) - - await requestCancelledEvent.wait().wait(timeout = chronos.seconds(5)) - - await advanceToNextPeriod() - - let slotSize = slotSize(blocks, ecNodes, ecTolerance) - let pricePerSlotPerSecond = minPricePerBytePerSecond * slotSize - - check eventually ( - let endBalanceProvider = (await token.balanceOf(provider.ethAccount)) - endBalanceProvider > startBalanceProvider and - endBalanceProvider < startBalanceProvider + expiry.u256 * pricePerSlotPerSecond - ) - check eventually( - ( - let endBalanceClient = (await token.balanceOf(client.ethAccount)) - let endBalanceProvider = (await token.balanceOf(provider.ethAccount)) - (startBalanceClient - endBalanceClient) == - (endBalanceProvider - startBalanceProvider) - ), - timeout = 10 * 1000, # give client a bit of time to withdraw its funds - ) - - await slotFilledSubscription.unsubscribe() - await requestCancelledSubscription.unsubscribe() - - test "the collateral is returned after a sale is ignored", - NodeConfigs( - hardhat: HardhatConfig.none, - clients: CodexConfigs.init(nodes = 1).some, - providers: CodexConfigs.init(nodes = 3) - # .debug() - # uncomment to enable console log output - # .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics( - # "node", "marketplace", "sales", "reservations", "statemachine" - # ) - .some, - ): - let data = await RandomChunker.example(blocks = blocks) - let client0 = clients()[0] - let provider0 = providers()[0] - let provider1 = providers()[1] - let provider2 = providers()[2] - let duration = 20 * 60.uint64 - let slotSize = slotSize(blocks, ecNodes, ecTolerance) - - # Here we create 3 SP which can host 3 slot. - # While they will process the slot, each SP will - # create a reservation for each slot. - # Likely we will have 1 slot by SP and the other reservations - # will be ignored. In that case, the collateral assigned for - # the reservation should return to the availability. - discard await provider0.client.postAvailability( - totalSize = 3 * slotSize.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = 3 * slotSize * minPricePerBytePerSecond, - ) - discard await provider1.client.postAvailability( - totalSize = 3 * slotSize.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = 3 * slotSize * minPricePerBytePerSecond, - ) - discard await provider2.client.postAvailability( - totalSize = 3 * slotSize.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = 3 * slotSize * minPricePerBytePerSecond, - ) - - let cid = (await client0.client.upload(data)).get - - let purchaseId = await client0.client.requestStorage( - cid, - duration = duration, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 1.u256, - expiry = 10 * 60.uint64, - collateralPerByte = collateralPerByte, - nodes = ecNodes, - tolerance = ecTolerance, - ) - - let requestId = (await client0.client.requestId(purchaseId)).get - - discard await waitForRequestToStart() - - # Here we will check that for each provider, the total remaining collateral - # will match the available slots. - # So if a SP hosts 1 slot, it should have enough total remaining collateral - # to host 2 more slots. - for provider in providers(): - let client = provider.client - check eventually( - block: - let availabilities = (await client.getAvailabilities()).get - let availability = availabilities[0] - let slots = (await client.getSlots()).get - let availableSlots = (3 - slots.len).u256 - - availability.totalRemainingCollateral == - availableSlots * slotSize * minPricePerBytePerSecond, - timeout = 30 * 1000, - ) diff --git a/tests/integration/30_minutes/testproofs.nim.ignore b/tests/integration/30_minutes/testproofs.nim.ignore deleted file mode 100644 index b06e4d82..00000000 --- a/tests/integration/30_minutes/testproofs.nim.ignore +++ /dev/null @@ -1,356 +0,0 @@ -from std/times import inMilliseconds -import pkg/questionable -import pkg/codex/logutils -import ../../contracts/time -import ../../contracts/deployment -import ../../codex/helpers -import ../../examples -import ../marketplacesuite -import ../nodeconfigs - -export logutils - -logScope: - topics = "integration test proofs" - -marketplacesuite(name = "Hosts submit regular proofs", stopOnRequestFail = false): - const minPricePerBytePerSecond = 1.u256 - const collateralPerByte = 1.u256 - const blocks = 8 - const ecNodes = 3 - const ecTolerance = 1 - - test "hosts submit periodic proofs for slots they fill", - NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: HardhatConfig.none, - clients: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node, marketplace") - .some, - providers: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node", "clock") - .some, - ): - let client0 = clients()[0].client - let expiry = 10.periods - let duration = expiry + 5.periods - - let data = await RandomChunker.example(blocks = blocks) - let datasetSize = - datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - await createAvailabilities( - datasetSize.truncate(uint64), - duration, - collateralPerByte, - minPricePerBytePerSecond, - ) - - let cid = (await client0.upload(data)).get - - let purchaseId = await client0.requestStorage( - cid, - expiry = expiry, - duration = duration, - nodes = ecNodes, - tolerance = ecTolerance, - ) - - let purchase = (await client0.getPurchase(purchaseId)).get - check purchase.error == none string - - let slotSize = slotSize(blocks, ecNodes, ecTolerance) - - discard await waitForRequestToStart(expiry.int) - - var proofWasSubmitted = false - proc onProofSubmitted(event: ?!ProofSubmitted) = - proofWasSubmitted = event.isOk - - let subscription = await marketplace.subscribe(ProofSubmitted, onProofSubmitted) - - check eventually(proofWasSubmitted, timeout = (duration - expiry).int * 1000) - - await subscription.unsubscribe() - -marketplacesuite(name = "Simulate invalid proofs", stopOnRequestFail = false): - # TODO: these are very loose tests in that they are not testing EXACTLY how - # proofs were marked as missed by the validator. These tests should be - # tightened so that they are showing, as an integration test, that specific - # proofs are being marked as missed by the validator. - - const minPricePerBytePerSecond = 1.u256 - const collateralPerByte = 1.u256 - const blocks = 8 - const ecNodes = 3 - const ecTolerance = 1 - - test "slot is freed after too many invalid proofs submitted", - NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: HardhatConfig.none, - clients: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "clock") - .some, - providers: CodexConfigs - .init(nodes = 1) - .withSimulateProofFailures(idx = 0, failEveryNProofs = 1) - # .debug() - # uncomment to enable console log output - # .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics( - # "marketplace", "sales", "reservations", "node", "clock", "slotsbuilder" - # ) - .some, - validators: CodexConfigs.init(nodes = 1) - # .debug() - # uncomment to enable console log output - # .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("validator", "onchain", "ethers", "clock") - .some, - ): - let client0 = clients()[0].client - let expiry = 10.periods - let duration = expiry + 10.periods - - let data = await RandomChunker.example(blocks = blocks) - let datasetSize = - datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - await createAvailabilities( - datasetSize.truncate(uint64), - duration, - collateralPerByte, - minPricePerBytePerSecond, - ) - - let cid = (await client0.upload(data)).get - - let purchaseId = ( - await client0.requestStorage( - cid, - expiry = expiry, - duration = duration, - nodes = ecNodes, - tolerance = ecTolerance, - proofProbability = 1.u256, - ) - ) - let requestId = (await client0.requestId(purchaseId)).get - - discard await waitForRequestToStart(expiry.int) - - var slotWasFreed = false - proc onSlotFreed(event: ?!SlotFreed) = - if event.isOk and event.value.requestId == requestId: - slotWasFreed = true - - let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - check eventually(slotWasFreed, timeout = (duration - expiry).int * 1000) - - await subscription.unsubscribe() - - test "slot is not freed when not enough invalid proofs submitted", - NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: HardhatConfig.none, - clients: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node", "clock") - .some, - providers: CodexConfigs - .init(nodes = 1) - .withSimulateProofFailures(idx = 0, failEveryNProofs = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node") - .some, - validators: CodexConfigs.init(nodes = 1) - # .debug() - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("validator", "onchain", "ethers", "clock") - .some, - ): - let client0 = clients()[0].client - let expiry = 10.periods - let duration = expiry + 10.periods - - let data = await RandomChunker.example(blocks = blocks) - let datasetSize = - datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - await createAvailabilities( - datasetSize.truncate(uint64), - duration, - collateralPerByte, - minPricePerBytePerSecond, - ) - - let cid = (await client0.upload(data)).get - - let purchaseId = await client0.requestStorage( - cid, - expiry = expiry, - duration = duration, - nodes = ecNodes, - tolerance = ecTolerance, - proofProbability = 1.u256, - ) - let requestId = (await client0.requestId(purchaseId)).get - - var slotWasFilled = false - proc onSlotFilled(eventResult: ?!SlotFilled) = - assert not eventResult.isErr - let event = !eventResult - - if event.requestId == requestId: - slotWasFilled = true - - let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - - # wait for the first slot to be filled - check eventually(slotWasFilled, timeout = expiry.int * 1000) - - var slotWasFreed = false - proc onSlotFreed(event: ?!SlotFreed) = - if event.isOk and event.value.requestId == requestId: - slotWasFreed = true - - let freedSubscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - # In 2 periods you cannot have enough invalid proofs submitted: - await sleepAsync(2.periods.int.seconds) - check not slotWasFreed - - await filledSubscription.unsubscribe() - await freedSubscription.unsubscribe() - - # TODO: uncomment once fixed - # WARNING: in the meantime minPrice has changed to minPricePerBytePerSecond - # and maxCollateral has changed to totalCollateral - double check if - # it is set correctly below - # test "host that submits invalid proofs is paid out less", NodeConfigs( - # # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - # # hardhat: HardhatConfig().withLogFile(), - - # clients: - # CodexConfig() - # .nodes(1) - # # .debug() # uncomment to enable console log output.debug() - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "erasure", "clock", "purchases"), - - # providers: - # CodexConfig() - # .nodes(3) - # .simulateProofFailuresFor(providerIdx=0, failEveryNProofs=2) - # # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("marketplace", "sales", "reservations", "node"), - - # validators: - # CodexConfig() - # .nodes(1) - # # .debug() - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("validator") - # ): - # let client0 = clients()[0].client - # let provider0 = providers()[0] - # let provider1 = providers()[1] - # let provider2 = providers()[2] - # let totalPeriods = 25 - - # let datasetSizeInBlocks = 3 - # let data = await RandomChunker.example(blocks=datasetSizeInBlocks) - # # original data = 3 blocks so slot size will be 4 blocks - # let slotSize = (DefaultBlockSize * 4.NBytes).Natural.u256 - - # discard provider0.client.postAvailability( - # totalSize=slotSize, # should match 1 slot only - # duration=totalPeriods.periods.u256, - # minPricePerBytePerSecond=minPricePerBytePerSecond, - # totalCollateral=slotSize * minPricePerBytePerSecond, - # enabled = true.some, - # until = 0.SecondsSince1970.some, - # ) - - # let cid = client0.upload(data).get - - # let purchaseId = await client0.requestStorage( - # cid, - # duration=totalPeriods.periods, - # expiry=10.periods, - # nodes=3, - # tolerance=1, - # origDatasetSizeInBlocks=datasetSizeInBlocks - # ) - - # without requestId =? client0.requestId(purchaseId): - # fail() - - # var filledSlotIds: seq[SlotId] = @[] - # proc onSlotFilled(event: SlotFilled) = - # let slotId = slotId(event.requestId, event.slotIndex) - # filledSlotIds.add slotId - - # let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - - # # wait til first slot is filled - # check eventually filledSlotIds.len > 0 - - # # now add availability for providers 1 and 2, which should allow them to to - # # put the remaining slots in their queues - # discard provider1.client.postAvailability( - # totalSize=slotSize, # should match 1 slot only - # duration=totalPeriods.periods.u256, - # minPricePerBytePerSecond=minPricePerBytePerSecond, - # totalCollateral=slotSize * minPricePerBytePerSecond - # ) - - # check eventually filledSlotIds.len > 1 - - # discard provider2.client.postAvailability( - # totalSize=slotSize, # should match 1 slot only - # duration=totalPeriods.periods.u256, - # minPricePerBytePerSecond=minPricePerBytePerSecond, - # totalCollateral=slotSize * minPricePerBytePerSecond - # ) - - # check eventually filledSlotIds.len > 2 - - # # Wait til second slot is filled. SaleFilled happens too quickly, check SaleProving instead. - # check eventually provider1.client.saleStateIs(filledSlotIds[1], "SaleProving") - # check eventually provider2.client.saleStateIs(filledSlotIds[2], "SaleProving") - - # check eventually client0.purchaseStateIs(purchaseId, "started") - - # let currentPeriod = await getCurrentPeriod() - # check eventuallyP( - # # SaleFinished happens too quickly, check SalePayout instead - # provider0.client.saleStateIs(filledSlotIds[0], "SalePayout"), - # currentPeriod + totalPeriods.u256 + 1) - - # check eventuallyP( - # # SaleFinished happens too quickly, check SalePayout instead - # provider1.client.saleStateIs(filledSlotIds[1], "SalePayout"), - # currentPeriod + totalPeriods.u256 + 1) - - # check eventuallyP( - # # SaleFinished happens too quickly, check SalePayout instead - # provider2.client.saleStateIs(filledSlotIds[2], "SalePayout"), - # currentPeriod + totalPeriods.u256 + 1) - - # check eventually( - # (await token.balanceOf(provider1.ethAccount)) > - # (await token.balanceOf(provider0.ethAccount)) - # ) - - # await subscription.unsubscribe() diff --git a/tests/integration/30_minutes/testslotrepair.nim.ignore b/tests/integration/30_minutes/testslotrepair.nim.ignore deleted file mode 100644 index f7d8dba7..00000000 --- a/tests/integration/30_minutes/testslotrepair.nim.ignore +++ /dev/null @@ -1,309 +0,0 @@ -import pkg/questionable -import pkg/codex/logutils -import ../../contracts/time -import ../../contracts/deployment -import ../../codex/helpers -import ../../examples -import ../marketplacesuite -import ../nodeconfigs - -export logutils - -logScope: - topics = "integration test slot repair" - -marketplacesuite(name = "SP Slot Repair", stopOnRequestFail = true): - const minPricePerBytePerSecond = 1.u256 - const collateralPerByte = 1.u256 - const blocks = 3 - const ecNodes = 3 - const ecTolerance = 1 - const size = slotSize(blocks, ecNodes, ecTolerance) - - var filledSlotIds: seq[SlotId] = @[] - var freedSlotId = none SlotId - var requestId: RequestId - - # Here we are keeping track of the slot filled using their ids. - proc onSlotFilled(eventResult: ?!SlotFilled) = - assert not eventResult.isErr - let event = !eventResult - - if event.requestId == requestId: - let slotId = slotId(event.requestId, event.slotIndex) - filledSlotIds.add slotId - - # Here we are retrieving the slot id freed. - # When the event is triggered, the slot id is removed - # from the filled slot id list. - proc onSlotFreed(eventResult: ?!SlotFreed) = - assert not eventResult.isErr - let event = !eventResult - let slotId = slotId(event.requestId, event.slotIndex) - - if event.requestId == requestId: - assert slotId in filledSlotIds - filledSlotIds.del(filledSlotIds.find(slotId)) - freedSlotId = some(slotId) - - proc createPurchase(client: CodexClient): Future[PurchaseId] {.async.} = - let data = await RandomChunker.example(blocks = blocks) - let cid = (await client.upload(data)).get - - let purchaseId = await client.requestStorage( - cid, - expiry = 10.periods, - duration = 20.periods, - nodes = ecNodes, - tolerance = ecTolerance, - collateralPerByte = 1.u256, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 1.u256, - ) - requestId = (await client.requestId(purchaseId)).get - - return purchaseId - - proc freeSlot(provider: CodexClient): Future[void] {.async.} = - # Get the second provider signer. - let signer = ethProvider.getSigner(accounts[2]) - let marketplaceWithSecondProviderSigner = marketplace.connect(signer) - - # Call freeSlot to speed up the process. - # It accelerates the test by skipping validator - # proof verification and not waiting for the full period. - # The downside is that this doesn't reflect the real slot freed process. - let slots = (await provider.getSlots()).get() - let slotId = slotId(requestId, slots[0].slotIndex) - discard await marketplaceWithSecondProviderSigner.freeSlot(slotId) - - setup: - filledSlotIds = @[] - freedSlotId = none SlotId - - test "repair from local store", - NodeConfigs( - clients: CodexConfigs.init(nodes = 1).some, - # .debug() - # .withLogFile() - # .withLogTopics("node", "erasure").some, - providers: CodexConfigs - .init(nodes = 2) - .withSimulateProofFailures(idx = 1, failEveryNProofs = 1) - # .debug() - .withLogFile() - .withLogTopics("marketplace", "sales", "reservations", "statemachine").some, - validators: CodexConfigs.init(nodes = 1).some, - # .debug() - # .withLogFile() - # .withLogTopics("validator").some, - ): - let client0 = clients()[0] - let provider0 = providers()[0] - let provider1 = providers()[1] - let expiry = 10.periods - let duration = 20.periods - - # Let's create 2 availabilities - # SP 1 will hosts 2 slots - # SP 2 will hosts 1 slot - let availability0 = ( - await provider0.client.postAvailability( - totalSize = 2 * size.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = 3 * size * collateralPerByte, - ) - ).get - let availability1 = ( - await provider1.client.postAvailability( - totalSize = size.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size * collateralPerByte, - ) - ).get - - let purchaseId = await createPurchase(client0.client) - - let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - # Wait for purchase starts, meaning that the slots are filled. - discard await waitForRequestToStart(expiry.int) - - # stop client so it doesn't serve any blocks anymore - await client0.stop() - - # Let's disable the second availability, - # SP 2 will not pick the slot again. - await provider1.client.patchAvailability( - availabilityId = availability1.id, enabled = false.some - ) - - # Update the size of the availability for the SP 1, - # he will repair and host the freed slot - await provider0.client.patchAvailability( - availabilityId = availability0.id, - totalSize = (3 * size.truncate(uint64)).uint64.some, - ) - - # Let's free the slot to speed up the process - await freeSlot(provider1.client) - - # We expect that the freed slot is added in the filled slot id list, - # meaning that the slot was repaired locally by SP 1. - check eventually( - freedSlotId.get in filledSlotIds, timeout = (duration - expiry).int * 1000 - ) - - await filledSubscription.unsubscribe() - await slotFreedsubscription.unsubscribe() - - test "repair from local and remote store", - NodeConfigs( - clients: CodexConfigs.init(nodes = 1) - # .debug() - # .withLogTopics("node", "erasure") - .some, - providers: CodexConfigs.init(nodes = 3) - # .debug() - # .withLogFile() - # .withLogTopics("marketplace", "sales", "statemachine", "reservations") - .some, - ): - let client0 = clients()[0] - let provider0 = providers()[0] - let provider1 = providers()[1] - let provider2 = providers()[2] - let expiry = 10.periods - let duration = 20.periods - - # SP 1, SP 2 and SP 3 will host one slot - let availability0 = ( - await provider0.client.postAvailability( - totalSize = size.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size * collateralPerByte, - ) - ).get - let availability1 = ( - await provider1.client.postAvailability( - totalSize = size.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size * collateralPerByte, - ) - ).get - discard await provider2.client.postAvailability( - totalSize = size.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size * collateralPerByte, - ) - - let purchaseId = await createPurchase(client0.client) - - let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - # Wait for purchase starts, meaning that the slots are filled. - discard await waitForRequestToStart(expiry.int) - - # stop client so it doesn't serve any blocks anymore - await client0.stop() - - # Let's disable the availability, - # SP 2 will not pick the slot again. - await provider1.client.patchAvailability(availability1.id, enabled = false.some) - - # Update the size of the availability for the SP 1, - # he will repair and host the freed slot - await provider0.client.patchAvailability( - availability0.id, - totalSize = (2 * size.truncate(uint64)).some, - totalCollateral = (2 * size * collateralPerByte).some, - ) - - # Let's free the slot to speed up the process - await freeSlot(provider1.client) - - # We expect that the freed slot is added in the filled slot id list, - # meaning that the slot was repaired locally and remotely (using SP 3) by SP 1. - check eventually(freedSlotId.isSome, timeout = expiry.int * 1000) - check eventually(freedSlotId.get in filledSlotIds, timeout = expiry.int * 1000) - - await filledSubscription.unsubscribe() - await slotFreedsubscription.unsubscribe() - - test "repair from remote store only", - NodeConfigs( - clients: CodexConfigs.init(nodes = 1) - # .debug() - # .withLogFile() - # .withLogTopics("node", "erasure") - .some, - providers: CodexConfigs.init(nodes = 3) - # .debug() - # .withLogFile() - # .withLogTopics("marketplace", "sales", "statemachine", "reservations") - .some, - ): - let client0 = clients()[0] - let provider0 = providers()[0] - let provider1 = providers()[1] - let provider2 = providers()[2] - let expiry = 10.periods - let duration = expiry + 10.periods - - # SP 1 will host 2 slots - # SP 2 will host 1 slot - discard await provider0.client.postAvailability( - totalSize = 2 * size.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = 2 * size * collateralPerByte, - ) - let availability1 = ( - await provider1.client.postAvailability( - totalSize = size.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size * collateralPerByte, - ) - ).get - - let purchaseId = await createPurchase(client0.client) - - let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) - let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed) - - # Wait for purchase starts, meaning that the slots are filled. - discard await waitForRequestToStart(expiry.int) - - # stop client so it doesn't serve any blocks anymore - await client0.stop() - - # Let's create an availability for SP3, - # he will host the repaired slot. - discard await provider2.client.postAvailability( - totalSize = size.truncate(uint64), - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size * collateralPerByte, - ) - - # Let's disable the availability, - # SP 2 will not pick the slot again. - await provider1.client.patchAvailability(availability1.id, enabled = false.some) - - # Let's free the slot to speed up the process - await freeSlot(provider1.client) - - # At this point, SP 3 should repair the slot from SP 1 and host it. - check eventually(freedSlotId.isSome, timeout = expiry.int * 1000) - check eventually(freedSlotId.get in filledSlotIds, timeout = expiry.int * 1000) - - await filledSubscription.unsubscribe() - await slotFreedsubscription.unsubscribe() diff --git a/tests/integration/30_minutes/testvalidator.nim.ignore b/tests/integration/30_minutes/testvalidator.nim.ignore deleted file mode 100644 index ed67b5d0..00000000 --- a/tests/integration/30_minutes/testvalidator.nim.ignore +++ /dev/null @@ -1,177 +0,0 @@ -from std/times import inMilliseconds, initDuration, inSeconds, fromUnix -import std/sugar -import pkg/codex/logutils -import pkg/questionable/results -import pkg/ethers/provider -import ../../contracts/time -import ../../contracts/deployment -import ../../codex/helpers -import ../../examples -import ../marketplacesuite -import ../nodeconfigs - -export logutils - -logScope: - topics = "integration test validation" - -marketplacesuite(name = "Validation", stopOnRequestFail = false): - const blocks = 8 - const ecNodes = 3 - const ecTolerance = 1 - const proofProbability = 1.u256 - - const collateralPerByte = 1.u256 - const minPricePerBytePerSecond = 1.u256 - - test "validator marks proofs as missing when using validation groups", - NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: HardhatConfig.none, - clients: CodexConfigs - .init(nodes = 1) - # .debug() # uncomment to enable console log output - .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("purchases", "onchain").some, - providers: CodexConfigs - .init(nodes = 1) - .withSimulateProofFailures(idx = 0, failEveryNProofs = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("sales", "onchain") - .some, - validators: CodexConfigs - .init(nodes = 2) - .withValidationGroups(groups = 2) - .withValidationGroupIndex(idx = 0, groupIndex = 0) - .withValidationGroupIndex(idx = 1, groupIndex = 1) - # .debug() # uncomment to enable console log output - .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("validator") - # each topic as a separate string argument - .some, - ): - let client0 = clients()[0].client - let expiry = 5.periods - let duration = expiry + 10.periods - - # let mine a block to sync the blocktime with the current clock - discard await ethProvider.send("evm_mine") - - var currentTime = await ethProvider.currentTime() - let requestEndTime = currentTime.truncate(uint64) + duration - - let data = await RandomChunker.example(blocks = blocks) - let datasetSize = - datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - await createAvailabilities( - datasetSize.truncate(uint64), - duration, - collateralPerByte, - minPricePerBytePerSecond, - ) - - let cid = (await client0.upload(data)).get - let purchaseId = await client0.requestStorage( - cid, - expiry = expiry, - duration = duration, - nodes = ecNodes, - tolerance = ecTolerance, - proofProbability = proofProbability, - ) - let requestId = (await client0.requestId(purchaseId)).get - - debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - - discard await waitForRequestToStart(expiry.int + 60) - - discard await ethProvider.send("evm_mine") - currentTime = await ethProvider.currentTime() - let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int - - debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - - discard await waitForRequestToFail(secondsTillRequestEnd + 60) - - test "validator uses historical state to mark missing proofs", - NodeConfigs( - # Uncomment to start Hardhat automatically, typically so logs can be inspected locally - hardhat: HardhatConfig.none, - clients: CodexConfigs - .init(nodes = 1) - # .debug() # uncomment to enable console log output - .withLogFile() - # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("purchases", "onchain").some, - providers: CodexConfigs - .init(nodes = 1) - .withSimulateProofFailures(idx = 0, failEveryNProofs = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("sales", "onchain") - .some, - ): - let client0 = clients()[0].client - let expiry = 5.periods - let duration = expiry + 10.periods - - # let mine a block to sync the blocktime with the current clock - discard await ethProvider.send("evm_mine") - - var currentTime = await ethProvider.currentTime() - let requestEndTime = currentTime.truncate(uint64) + duration - - let data = await RandomChunker.example(blocks = blocks) - let datasetSize = - datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance) - await createAvailabilities( - datasetSize.truncate(uint64), - duration, - collateralPerByte, - minPricePerBytePerSecond, - ) - - let cid = (await client0.upload(data)).get - let purchaseId = await client0.requestStorage( - cid, - expiry = expiry, - duration = duration, - nodes = ecNodes, - tolerance = ecTolerance, - proofProbability = proofProbability, - ) - let requestId = (await client0.requestId(purchaseId)).get - - debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId - - discard await waitForRequestToStart(expiry.int + 60) - - # extra block just to make sure we have one that separates us - # from the block containing the last (past) SlotFilled event - discard await ethProvider.send("evm_mine") - - var validators = CodexConfigs - .init(nodes = 2) - .withValidationGroups(groups = 2) - .withValidationGroupIndex(idx = 0, groupIndex = 0) - .withValidationGroupIndex(idx = 1, groupIndex = 1) - # .debug() # uncomment to enable console log output - .withLogFile() - # uncomment to output log file to: # tests/integration/logs/ //_.log - .withLogTopics("validator") # each topic as a separate string argument - - failAndTeardownOnError "failed to start validator nodes": - for config in validators.configs.mitems: - let node = await startValidatorNode(config) - running.add RunningNode(role: Role.Validator, node: node) - - discard await ethProvider.send("evm_mine") - currentTime = await ethProvider.currentTime() - let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int - - debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds - - discard await waitForRequestToFail(secondsTillRequestEnd + 60) diff --git a/tests/integration/5_minutes/testrestapi.nim b/tests/integration/5_minutes/testrestapi.nim index 9a6b9c11..1903cce0 100644 --- a/tests/integration/5_minutes/testrestapi.nim +++ b/tests/integration/5_minutes/testrestapi.nim @@ -24,27 +24,6 @@ twonodessuite "REST API": check cid1 != cid2 - test "node shows used and available space", twoNodesConfig: - discard (await client1.upload("some file contents")).get - let totalSize = 12.uint64 - let minPricePerBytePerSecond = 1.u256 - let totalCollateral = totalSize.u256 * minPricePerBytePerSecond - discard ( - await client1.postAvailability( - totalSize = totalSize, - duration = 2.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - enabled = true.some, - ) - ).get - let space = (await client1.space()).tryGet() - check: - space.totalBlocks == 2 - space.quotaMaxBytes == 21474836480.NBytes - space.quotaUsedBytes == 65592.NBytes - space.quotaReservedBytes == 12.NBytes - test "node lists local files", twoNodesConfig: let content1 = "some file contents" let content2 = "some other contents" @@ -56,46 +35,6 @@ twonodessuite "REST API": check: [cid1, cid2].allIt(it in list.content.mapIt(it.cid)) - test "request storage succeeds for sufficiently sized datasets", twoNodesConfig: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client1.upload(data)).get - let response = ( - await client1.requestStorageRaw( - cid, - duration = 10.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - collateralPerByte = 1.u256, - expiry = 9.uint64, - ) - ) - - check: - response.status == 200 - - for ecParams in @[ - (minBlocks: 2, nodes: 3, tolerance: 1), (minBlocks: 3, nodes: 5, tolerance: 2) - ]: - let (minBlocks, nodes, tolerance) = ecParams - test "request storage succeeds if nodes and tolerance within range " & - fmt"({minBlocks=}, {nodes=}, {tolerance=})", twoNodesConfig: - let data = await RandomChunker.example(blocks = minBlocks) - let cid = (await client1.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - - var responseBefore = ( - await client1.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 200 - test "node accepts file uploads with content type", twoNodesConfig: let headers = @[("Content-Type", "text/plain")] let response = await client1.uploadRaw("some file contents", headers) diff --git a/tests/integration/5_minutes/testrestapivalidation.nim b/tests/integration/5_minutes/testrestapivalidation.nim index 6530f355..46141aae 100644 --- a/tests/integration/5_minutes/testrestapivalidation.nim +++ b/tests/integration/5_minutes/testrestapivalidation.nim @@ -1,7 +1,7 @@ import std/times -import pkg/ethers import pkg/codex/conf -import pkg/codex/contracts +import pkg/stint +from pkg/libp2p import Cid, `$` import ../../asynctest import ../../checktest import ../../examples @@ -17,88 +17,13 @@ multinodesuite "Rest API validation": setup: client = clients()[0].client - test "should return 422 when attempting delete of non-existing dataset", config: + test "should return 204 when attempting delete of non-existing dataset", config: let data = await RandomChunker.example(blocks = 2) let cid = (await client.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 0 - - var responseBefore = await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == "Tolerance needs to be bigger then zero" - - test "request storage fails for datasets that are too small", config: - let cid = (await client.upload("some file contents")).get - let response = ( - await client.requestStorageRaw( - cid, - duration = 10.uint64, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - collateralPerByte = 1.u256, - expiry = 9.uint64, - ) - ) - - check: - response.status == 422 - (await response.body) == - "Dataset too small for erasure parameters, need at least " & - $(2 * DefaultBlockSize.int) & " bytes" - - test "request storage fails if nodes and tolerance aren't correct", config: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let ecParams = @[(1, 1), (2, 1), (3, 2), (3, 3)] - - for ecParam in ecParams: - let (nodes, tolerance) = ecParam - - var responseBefore = ( - await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == - "Invalid parameters: parameters must satify `1 < (nodes - tolerance) ≥ tolerance`" - - test "request storage fails if tolerance > nodes (underflow protection)", config: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 0 - - var responseBefore = ( - await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == "Tolerance needs to be bigger then zero" + + let responseBefore = await client.deleteRaw($Cid.example) + check responseBefore.status == 204 + check (await responseBefore.body) == "" # No content test "upload fails if content disposition contains bad filename", config: let headers = @[("Content-Disposition", "attachment; filename=\"exam*ple.txt\"")] @@ -114,289 +39,6 @@ multinodesuite "Rest API validation": check response.status == 422 check (await response.body) == "The MIME type 'hello/world' is not valid." - test "updating non-existing availability", config: - let nonExistingResponse = await client.patchAvailabilityRaw( - AvailabilityId.example, - duration = 100.uint64.some, - minPricePerBytePerSecond = 2.u256.some, - totalCollateral = 200.u256.some, - ) - check nonExistingResponse.status == 404 - - test "updating availability - freeSize is not allowed to be changed", config: - let availability = ( - await client.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - ).get - let freeSizeResponse = - await client.patchAvailabilityRaw(availability.id, freeSize = 110000.uint64.some) - check freeSizeResponse.status == 422 - check "not allowed" in (await freeSizeResponse.body) - - test "creating availability above the node quota returns 422", config: - let response = await client.postAvailabilityRaw( - totalSize = 24000000000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - - check response.status == 422 - check (await response.body) == "Not enough storage quota" - - test "updating availability above the node quota returns 422", config: - let availability = ( - await client.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - ).get - let response = await client.patchAvailabilityRaw( - availability.id, totalSize = 24000000000.uint64.some - ) - - check response.status == 422 - check (await response.body) == "Not enough storage quota" - - test "creating availability when total size is zero returns 422", config: - let response = await client.postAvailabilityRaw( - totalSize = 0.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - - check response.status == 422 - check (await response.body) == "Total size must be larger then zero" - - test "updating availability when total size is zero returns 422", config: - let availability = ( - await client.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - ).get - let response = - await client.patchAvailabilityRaw(availability.id, totalSize = 0.uint64.some) - - check response.status == 422 - check (await response.body) == "Total size must be larger then zero" - - test "creating availability when total size is negative returns 422", config: - let json = - %*{ - "totalSize": "-1", - "duration": "200", - "minPricePerBytePerSecond": "3", - "totalCollateral": "300", - } - let response = await client.post(client.buildUrl("/sales/availability"), $json) - - check response.status == 400 - check (await response.body) == "Parsed integer outside of valid range" - - test "updating availability when total size is negative returns 422", config: - let availability = ( - await client.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - ).get - - let json = %*{"totalSize": "-1"} - let response = await client.patch( - client.buildUrl("/sales/availability/") & $availability.id, $json - ) - - check response.status == 400 - check (await response.body) == "Parsed integer outside of valid range" - - test "request storage fails if tolerance is zero", config: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 0 - - var responseBefore = ( - await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == "Tolerance needs to be bigger then zero" - - test "request storage fails if duration exceeds limit", config: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client.upload(data)).get - let duration = (31 * 24 * 60 * 60).uint64 - # 31 days TODO: this should not be hardcoded, but waits for https://github.com/logos-storage/logos-storage-nim/issues/1056 - let proofProbability = 3.u256 - let expiry = 30.uint - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 2 - let pricePerBytePerSecond = 1.u256 - - var responseBefore = ( - await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, - expiry, nodes.uint, tolerance.uint, - ) - ) - - check responseBefore.status == 422 - check "Duration exceeds limit of" in (await responseBefore.body) - - test "request storage fails if expiry is zero", config: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 0.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 1 - - var responseBefore = await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == - "Expiry must be greater than zero and less than the request's duration" - - test "request storage fails if proof probability is zero", config: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 0.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 1 - - var responseBefore = await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == "Proof probability must be greater than zero" - - test "request storage fails if price per byte per second is zero", config: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 0.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 1.u256 - let nodes = 3 - let tolerance = 1 - - var responseBefore = await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == - "Price per byte per second must be greater than zero" - - test "request storage fails if collareral per byte is zero", config: - let data = await RandomChunker.example(blocks = 2) - let cid = (await client.upload(data)).get - let duration = 100.uint64 - let pricePerBytePerSecond = 1.u256 - let proofProbability = 3.u256 - let expiry = 30.uint64 - let collateralPerByte = 0.u256 - let nodes = 3 - let tolerance = 1 - - var responseBefore = await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes.uint, tolerance.uint, - ) - - check responseBefore.status == 422 - check (await responseBefore.body) == "Collateral per byte must be greater than zero" - - test "creating availability fails when until is negative", config: - let totalSize = 12.uint64 - let minPricePerBytePerSecond = 1.u256 - let totalCollateral = totalSize.u256 * minPricePerBytePerSecond - let response = await client.postAvailabilityRaw( - totalSize = totalSize, - duration = 2.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - until = -1.SecondsSince1970.some, - ) - - check: - response.status == 422 - (await response.body) == "Cannot set until to a negative value" - - test "creating availability fails when duration is zero", config: - let response = await client.postAvailabilityRaw( - totalSize = 12.uint64, - duration = 0.uint64, - minPricePerBytePerSecond = 1.u256, - totalCollateral = 22.u256, - until = -1.SecondsSince1970.some, - ) - - check: - response.status == 422 - (await response.body) == "duration must be larger then zero" - - test "creating availability fails when minPricePerBytePerSecond is zero", config: - let response = await client.postAvailabilityRaw( - totalSize = 12.uint64, - duration = 1.uint64, - minPricePerBytePerSecond = 0.u256, - totalCollateral = 22.u256, - until = -1.SecondsSince1970.some, - ) - - check: - response.status == 422 - (await response.body) == "minPricePerBytePerSecond must be larger then zero" - - test "creating availability fails when totalCollateral is zero", config: - let response = await client.postAvailabilityRaw( - totalSize = 12.uint64, - duration = 1.uint64, - minPricePerBytePerSecond = 2.u256, - totalCollateral = 0.u256, - until = -1.SecondsSince1970.some, - ) - - check: - response.status == 422 - (await response.body) == "totalCollateral must be larger then zero" - test "has block returns error 400 when the cid is invalid", config: let response = await client.hasBlockRaw("invalid-cid") diff --git a/tests/integration/5_minutes/testsales.nim.ignore b/tests/integration/5_minutes/testsales.nim.ignore deleted file mode 100644 index 246d8fc7..00000000 --- a/tests/integration/5_minutes/testsales.nim.ignore +++ /dev/null @@ -1,233 +0,0 @@ -import std/httpclient -import std/times -import pkg/codex/contracts -from pkg/codex/stores/repostore/types import DefaultQuotaBytes -import ../twonodes -import ../../codex/examples -import ../../contracts/time -import ../codexconfig -import ../codexclient -import ../nodeconfigs -import ../marketplacesuite - -proc findItem[T](items: seq[T], item: T): ?!T = - for tmp in items: - if tmp == item: - return success tmp - - return failure("Not found") - -marketplacesuite(name = "Sales", stopOnRequestFail = true): - let salesConfig = NodeConfigs( - clients: CodexConfigs.init(nodes = 1).some, - providers: CodexConfigs.init(nodes = 1) - # .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "sales", "reservations", "node", "proving", "clock") - .some, - ) - - var host: CodexClient - var client: CodexClient - - setup: - host = providers()[0].client - client = clients()[0].client - - test "node handles new storage availability", salesConfig: - let availability1 = ( - await host.postAvailability( - totalSize = 1.uint64, - duration = 2.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 4.u256, - ) - ).get - let availability2 = ( - await host.postAvailability( - totalSize = 4.uint64, - duration = 5.uint64, - minPricePerBytePerSecond = 6.u256, - totalCollateral = 7.u256, - ) - ).get - check availability1 != availability2 - - test "node lists storage that is for sale", salesConfig: - let availability = ( - await host.postAvailability( - totalSize = 1.uint64, - duration = 2.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 4.u256, - ) - ).get - check availability in (await host.getAvailabilities()).get - - test "updating availability", salesConfig: - let availability = ( - await host.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - ).get - - var until = getTime().toUnix() - - await host.patchAvailability( - availability.id, - duration = 100.uint64.some, - minPricePerBytePerSecond = 2.u256.some, - totalCollateral = 200.u256.some, - enabled = false.some, - until = until.some, - ) - - let updatedAvailability = - ((await host.getAvailabilities()).get).findItem(availability).get - check updatedAvailability.duration == 100.uint64 - check updatedAvailability.minPricePerBytePerSecond == 2 - check updatedAvailability.totalCollateral == 200 - check updatedAvailability.totalSize == 140000.uint64 - check updatedAvailability.freeSize == 140000.uint64 - check updatedAvailability.enabled == false - check updatedAvailability.until == until - - test "updating availability - updating totalSize", salesConfig: - let availability = ( - await host.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - ).get - await host.patchAvailability(availability.id, totalSize = 100000.uint64.some) - - let updatedAvailability = - ((await host.getAvailabilities()).get).findItem(availability).get - check updatedAvailability.totalSize == 100000 - check updatedAvailability.freeSize == 100000 - - test "updating availability - updating totalSize does not allow bellow utilized", - salesConfig: - let originalSize = 0xFFFFFF.uint64 - let data = await RandomChunker.example(blocks = 8) - let minPricePerBytePerSecond = 3.u256 - let collateralPerByte = 1.u256 - let totalCollateral = originalSize.u256 * collateralPerByte - let availability = ( - await host.postAvailability( - totalSize = originalSize, - duration = 20 * 60.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - ) - ).get - - # Lets create storage request that will utilize some of the availability's space - let cid = (await client.upload(data)).get - let id = await client.requestStorage( - cid, - duration = 20 * 60.uint64, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 3.u256, - expiry = (10 * 60).uint64, - collateralPerByte = collateralPerByte, - nodes = 3, - tolerance = 1, - ) - - discard await waitForRequestToStart() - - let updatedAvailability = - ((await host.getAvailabilities()).get).findItem(availability).get - check updatedAvailability.totalSize != updatedAvailability.freeSize - - let utilizedSize = updatedAvailability.totalSize - updatedAvailability.freeSize - let totalSizeResponse = ( - await host.patchAvailabilityRaw( - availability.id, totalSize = (utilizedSize - 1).some - ) - ) - check totalSizeResponse.status == 422 - check "totalSize must be larger then current totalSize" in - (await totalSizeResponse.body) - - await host.patchAvailability( - availability.id, totalSize = (originalSize + 20000).some - ) - let newUpdatedAvailability = - ((await host.getAvailabilities()).get).findItem(availability).get - check newUpdatedAvailability.totalSize == originalSize + 20000 - check newUpdatedAvailability.freeSize - updatedAvailability.freeSize == 20000 - - test "updating availability fails with until negative", salesConfig: - let availability = ( - await host.postAvailability( - totalSize = 140000.uint64, - duration = 200.uint64, - minPricePerBytePerSecond = 3.u256, - totalCollateral = 300.u256, - ) - ).get - - let response = - await host.patchAvailabilityRaw(availability.id, until = -1.SecondsSince1970.some) - - check: - (await response.body) == "Cannot set until to a negative value" - - test "returns an error when trying to update the until date before an existing a request is finished", - salesConfig: - let size = 0xFFFFFF.uint64 - let data = await RandomChunker.example(blocks = 8) - let duration = 20 * 60.uint64 - let minPricePerBytePerSecond = 3.u256 - let collateralPerByte = 1.u256 - let ecNodes = 3.uint - let ecTolerance = 1.uint - - # host makes storage available - let availability = ( - await host.postAvailability( - totalSize = size, - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = size.u256 * minPricePerBytePerSecond, - ) - ).get - - # client requests storage - let cid = (await client.upload(data)).get - let id = ( - await client.requestStorage( - cid, - duration = duration, - pricePerBytePerSecond = minPricePerBytePerSecond, - proofProbability = 3.u256, - expiry = 10 * 60.uint64, - collateralPerByte = collateralPerByte, - nodes = ecNodes, - tolerance = ecTolerance, - ) - ).get - - discard await waitForRequestToStart() - - let purchase = (await client.getPurchase(id)).get - check purchase.error == none string - - let unixNow = getTime().toUnix() - let until = unixNow + 1.SecondsSince1970 - - let response = await host.patchAvailabilityRaw( - availabilityId = availability.id, until = until.some - ) - - check: - response.status == 422 - (await response.body) == - "Until parameter must be greater or equal to the longest currently hosted slot" diff --git a/tests/integration/5_minutes/testupdownload.nim b/tests/integration/5_minutes/testupdownload.nim index 0ac2001d..ec5054c8 100644 --- a/tests/integration/5_minutes/testupdownload.nim +++ b/tests/integration/5_minutes/testupdownload.nim @@ -63,8 +63,6 @@ twonodessuite "Uploads and downloads": check manifest["datasetSize"].getInt() == 18 check manifest.hasKey("blockSize") == true check manifest["blockSize"].getInt() == 65536 - check manifest.hasKey("protected") == true - check manifest["protected"].getBool() == false test "node allows downloading only manifest", twoNodesConfig: let content1 = "some file contents" diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 3fdf564f..c8b3eca5 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -6,11 +6,9 @@ import pkg/questionable/results import pkg/chronos/apps/http/[httpserver, shttpserver, httpclient, httptable] import pkg/codex/logutils import pkg/codex/rest/json -import pkg/codex/purchasing import pkg/codex/errors -import pkg/codex/sales/reservations -export purchasing, httptable, httpclient +export httptable, httpclient type CodexClient* = ref object baseurl: string @@ -217,224 +215,9 @@ proc space*( RestRepoStore.fromJson(await response.body) -proc requestStorageRaw*( - client: CodexClient, - cid: Cid, - duration: uint64, - pricePerBytePerSecond: UInt256, - proofProbability: UInt256, - collateralPerByte: UInt256, - expiry: uint64 = 0, - nodes: uint = 3, - tolerance: uint = 1, -): Future[HttpClientResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]) -.} = - ## Call request storage REST endpoint - ## - let url = client.baseurl & "/storage/request/" & $cid - let json = - %*{ - "duration": duration, - "pricePerBytePerSecond": pricePerBytePerSecond, - "proofProbability": proofProbability, - "collateralPerByte": collateralPerByte, - "nodes": nodes, - "tolerance": tolerance, - } - - if expiry != 0: - json["expiry"] = %($expiry) - - return client.post(url, $json) - -proc requestStorage*( - client: CodexClient, - cid: Cid, - duration: uint64, - pricePerBytePerSecond: UInt256, - proofProbability: UInt256, - expiry: uint64, - collateralPerByte: UInt256, - nodes: uint = 3, - tolerance: uint = 1, -): Future[?!PurchaseId] {.async: (raises: [CancelledError, HttpError]).} = - ## Call request storage REST endpoint - ## - let - response = await client.requestStorageRaw( - cid, duration, pricePerBytePerSecond, proofProbability, collateralPerByte, expiry, - nodes, tolerance, - ) - body = await response.body - - if response.status != 200: - doAssert(false, body) - PurchaseId.fromHex(body).catch - -proc getPurchase*( - client: CodexClient, purchaseId: PurchaseId -): Future[?!RestPurchase] {.async: (raises: [CancelledError, HttpError]).} = - let url = client.baseurl & "/storage/purchases/" & purchaseId.toHex - try: - let body = await client.getContent(url) - return RestPurchase.fromJson(body) - except CatchableError as e: - return failure e.msg - -proc getSalesAgent*( - client: CodexClient, slotId: SlotId -): Future[?!RestSalesAgent] {.async: (raises: [CancelledError, HttpError]).} = - let url = client.baseurl & "/sales/slots/" & slotId.toHex - try: - let body = await client.getContent(url) - return RestSalesAgent.fromJson(body) - except CatchableError as e: - return failure e.msg - -proc postAvailabilityRaw*( - client: CodexClient, - totalSize, duration: uint64, - minPricePerBytePerSecond, totalCollateral: UInt256, - enabled: ?bool = bool.none, - until: ?SecondsSince1970 = SecondsSince1970.none, -): Future[HttpClientResponseRef] {.async: (raises: [CancelledError, HttpError]).} = - ## Post sales availability endpoint - ## - let url = client.baseurl & "/sales/availability" - let json = - %*{ - "totalSize": totalSize, - "duration": duration, - "minPricePerBytePerSecond": minPricePerBytePerSecond, - "totalCollateral": totalCollateral, - "enabled": enabled, - "until": until, - } - return await client.post(url, $json) - -proc postAvailability*( - client: CodexClient, - totalSize, duration: uint64, - minPricePerBytePerSecond, totalCollateral: UInt256, - enabled: ?bool = bool.none, - until: ?SecondsSince1970 = SecondsSince1970.none, -): Future[?!Availability] {.async: (raises: [CancelledError, HttpError]).} = - let response = await client.postAvailabilityRaw( - totalSize = totalSize, - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - enabled = enabled, - until = until, - ) - - let body = await response.body - - doAssert response.status == 201, - "expected 201 Created, got " & $response.status & ", body: " & body - Availability.fromJson(body) - -proc patchAvailabilityRaw*( - client: CodexClient, - availabilityId: AvailabilityId, - totalSize, freeSize, duration: ?uint64 = uint64.none, - minPricePerBytePerSecond, totalCollateral: ?UInt256 = UInt256.none, - enabled: ?bool = bool.none, - until: ?SecondsSince1970 = SecondsSince1970.none, -): Future[HttpClientResponseRef] {. - async: (raw: true, raises: [CancelledError, HttpError]) -.} = - ## Updates availability - ## - let url = client.baseurl & "/sales/availability/" & $availabilityId - - # TODO: Optionalize macro does not keep `serialize` pragmas so we can't use `Optionalize(RestAvailability)` here. - var json = %*{} - - if totalSize =? totalSize: - json["totalSize"] = %totalSize - - if freeSize =? freeSize: - json["freeSize"] = %freeSize - - if duration =? duration: - json["duration"] = %duration - - if minPricePerBytePerSecond =? minPricePerBytePerSecond: - json["minPricePerBytePerSecond"] = %minPricePerBytePerSecond - - if totalCollateral =? totalCollateral: - json["totalCollateral"] = %totalCollateral - - if enabled =? enabled: - json["enabled"] = %enabled - - if until =? until: - json["until"] = %until - - client.patch(url, $json) - -proc patchAvailability*( - client: CodexClient, - availabilityId: AvailabilityId, - totalSize, duration: ?uint64 = uint64.none, - minPricePerBytePerSecond, totalCollateral: ?UInt256 = UInt256.none, - enabled: ?bool = bool.none, - until: ?SecondsSince1970 = SecondsSince1970.none, -): Future[void] {.async: (raises: [CancelledError, HttpError]).} = - let response = await client.patchAvailabilityRaw( - availabilityId, - totalSize = totalSize, - duration = duration, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - enabled = enabled, - until = until, - ) - doAssert response.status == 204, "expected No Content, got " & $response.status - -proc getAvailabilities*( - client: CodexClient -): Future[?!seq[Availability]] {.async: (raises: [CancelledError, HttpError]).} = - ## Call sales availability REST endpoint - let url = client.baseurl & "/sales/availability" - let body = await client.getContent(url) - seq[Availability].fromJson(body) - -proc getAvailabilityReservations*( - client: CodexClient, availabilityId: AvailabilityId -): Future[?!seq[Reservation]] {.async: (raises: [CancelledError, HttpError]).} = - ## Retrieves Availability's Reservations - let url = client.baseurl & "/sales/availability/" & $availabilityId & "/reservations" - let body = await client.getContent(url) - seq[Reservation].fromJson(body) - -proc purchaseStateIs*( - client: CodexClient, id: PurchaseId, state: string -): Future[bool] {.async: (raises: [CancelledError, HttpError]).} = - (await client.getPurchase(id)).option .? state == some state - -proc saleStateIs*( - client: CodexClient, id: SlotId, state: string -): Future[bool] {.async: (raises: [CancelledError, HttpError]).} = - (await client.getSalesAgent(id)).option .? state == some state - -proc requestId*( - client: CodexClient, id: PurchaseId -): Future[?RequestId] {.async: (raises: [CancelledError, HttpError]).} = - return (await client.getPurchase(id)).option .? requestId - proc buildUrl*(client: CodexClient, path: string): string = return client.baseurl & path -proc getSlots*( - client: CodexClient -): Future[?!seq[Slot]] {.async: (raises: [CancelledError, HttpError]).} = - let url = client.baseurl & "/sales/slots" - let body = await client.getContent(url) - seq[Slot].fromJson(body) - proc hasBlock*( client: CodexClient, cid: Cid ): Future[?!bool] {.async: (raises: [CancelledError, HttpError]).} = diff --git a/tests/integration/codexconfig.nim b/tests/integration/codexconfig.nim index 138ae274..707906ab 100644 --- a/tests/integration/codexconfig.nim +++ b/tests/integration/codexconfig.nim @@ -22,7 +22,6 @@ type CodexConfig* = object cliOptions: Table[StartUpCmd, Table[string, CliOption]] - cliPersistenceOptions: Table[PersistenceCmd, Table[string, CliOption]] debugEnabled*: bool CodexConfigError* = object of CatchableError @@ -65,19 +64,6 @@ proc buildConfig( ## TODO: remove once proper exception handling added to nim-confutils raiseCodexConfigError msg & e.msg.postFix -proc addCliOption*( - config: var CodexConfig, group = PersistenceCmd.noCmd, cliOption: CliOption -) {.raises: [CodexConfigError].} = - var options = config.cliPersistenceOptions.getOrDefault(group) - options[cliOption.key] = cliOption # overwrite if already exists - config.cliPersistenceOptions[group] = options - discard config.buildConfig("Invalid cli arg " & $cliOption) - -proc addCliOption*( - config: var CodexConfig, group = PersistenceCmd.noCmd, key: string, value = "" -) {.raises: [CodexConfigError].} = - config.addCliOption(group, CliOption(key: key, value: value)) - proc addCliOption*( config: var CodexConfig, group = StartUpCmd.noCmd, cliOption: CliOption ) {.raises: [CodexConfigError].} = @@ -114,13 +100,6 @@ proc cliArgs*(config: CodexConfig): seq[string] {.gcsafe, raises: [CodexConfigEr var opts = config.cliOptions[cmd].values.toSeq args = args.concat(opts.map(o => $o)) - for cmd in PersistenceCmd: - if config.cliPersistenceOptions.hasKey(cmd): - if cmd != PersistenceCmd.noCmd: - args.add $cmd - var opts = config.cliPersistenceOptions[cmd].values.toSeq - args = args.concat(opts.map(o => $o)) - return args proc logFile*(config: CodexConfig): ?string {.raises: [CodexConfigError].} = @@ -248,65 +227,6 @@ proc withBlockMaintenanceInterval*( config.addCliOption("--block-mi", $interval) return startConfig -proc withSimulateProofFailures*( - self: CodexConfigs, idx: int, failEveryNProofs: int -): CodexConfigs {.raises: [CodexConfigError].} = - self.checkBounds idx - - var startConfig = self - startConfig.configs[idx].addCliOption( - StartUpCmd.persistence, "--simulate-proof-failures", $failEveryNProofs - ) - return startConfig - -proc withSimulateProofFailures*( - self: CodexConfigs, failEveryNProofs: int -): CodexConfigs {.raises: [CodexConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption( - StartUpCmd.persistence, "--simulate-proof-failures", $failEveryNProofs - ) - return startConfig - -proc withValidationGroups*( - self: CodexConfigs, groups: ValidationGroups -): CodexConfigs {.raises: [CodexConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption(StartUpCmd.persistence, "--validator-groups", $(groups)) - return startConfig - -proc withValidationGroupIndex*( - self: CodexConfigs, idx: int, groupIndex: uint16 -): CodexConfigs {.raises: [CodexConfigError].} = - self.checkBounds idx - - var startConfig = self - startConfig.configs[idx].addCliOption( - StartUpCmd.persistence, "--validator-group-index", $groupIndex - ) - return startConfig - -proc withEthProvider*( - self: CodexConfigs, idx: int, ethProvider: string -): CodexConfigs {.raises: [CodexConfigError].} = - self.checkBounds idx - - var startConfig = self - startConfig.configs[idx].addCliOption( - StartUpCmd.persistence, "--eth-provider", ethProvider - ) - return startConfig - -proc withEthProvider*( - self: CodexConfigs, ethProvider: string -): CodexConfigs {.raises: [CodexConfigError].} = - var startConfig = self - for config in startConfig.configs.mitems: - config.addCliOption(StartUpCmd.persistence, "--eth-provider", ethProvider) - return startConfig - proc logLevelWithTopics( config: CodexConfig, topics: varargs[string] ): string {.raises: [CodexConfigError].} = diff --git a/tests/integration/codexprocess.nim b/tests/integration/codexprocess.nim index 04c2904f..b5963aa3 100644 --- a/tests/integration/codexprocess.nim +++ b/tests/integration/codexprocess.nim @@ -43,12 +43,6 @@ proc dataDir(node: CodexProcess): string = let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) return config.dataDir.string -proc ethAccount*(node: CodexProcess): Address = - let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) - without ethAccount =? config.ethAccount: - raiseAssert "eth account not set" - return Address(ethAccount) - proc apiUrl*(node: CodexProcess): string = let config = CodexConf.load(cmdLine = node.arguments, quitOnFailure = false) return diff --git a/tests/integration/hardhatconfig.nim b/tests/integration/hardhatconfig.nim deleted file mode 100644 index 5de5bbc5..00000000 --- a/tests/integration/hardhatconfig.nim +++ /dev/null @@ -1,14 +0,0 @@ -type HardhatConfig* = object - logFile*: bool - debugEnabled*: bool - -proc debug*(self: HardhatConfig, enabled = true): HardhatConfig = - ## output log in stdout - var config = self - config.debugEnabled = enabled - return config - -proc withLogFile*(self: HardhatConfig, logToFile: bool = true): HardhatConfig = - var config = self - config.logFile = logToFile - return config diff --git a/tests/integration/hardhatprocess.nim b/tests/integration/hardhatprocess.nim deleted file mode 100644 index 915c8c53..00000000 --- a/tests/integration/hardhatprocess.nim +++ /dev/null @@ -1,145 +0,0 @@ -import pkg/questionable -import pkg/questionable/results -import pkg/confutils -import pkg/chronicles -import pkg/chronos -import pkg/chronos/asyncproc -import pkg/stew/io2 -import std/os -import std/sets -import std/sequtils -import std/strutils -import pkg/codex/conf -import pkg/codex/utils/trackedfutures -import ./codexclient -import ./nodeprocess - -export codexclient -export chronicles - -logScope: - topics = "integration testing hardhat process" - nodeName = "hardhat" - -type HardhatProcess* = ref object of NodeProcess - logFile: ?IoHandle - -method workingDir(node: HardhatProcess): string = - return - currentSourcePath() / ".." / ".." / ".." / "vendor" / "logos-storage-contracts-eth" - -method executable(node: HardhatProcess): string = - return "node_modules" / ".bin" / "hardhat" - -method startedOutput(node: HardhatProcess): string = - return "Started HTTP and WebSocket JSON-RPC server at" - -method processOptions(node: HardhatProcess): set[AsyncProcessOption] = - return {} - -method outputLineEndings(node: HardhatProcess): string {.raises: [].} = - return "\n" - -proc openLogFile(node: HardhatProcess, logFilePath: string): IoHandle = - let logFileHandle = - openFile(logFilePath, {OpenFlags.Write, OpenFlags.Create, OpenFlags.Truncate}) - - without fileHandle =? logFileHandle: - fatal "failed to open log file", - path = logFilePath, errorCode = $logFileHandle.error - - raiseAssert "failed to open log file, aborting" - - return fileHandle - -method start*(node: HardhatProcess) {.async.} = - let poptions = node.processOptions + {AsyncProcessOption.StdErrToStdOut} - - trace "starting node", - args = node.arguments, - executable = node.executable, - workingDir = node.workingDir, - processOptions = poptions - - try: - node.process = await startProcess( - node.executable, - node.workingDir, - @["node"].concat(node.arguments), - options = poptions, - stdoutHandle = AsyncProcess.Pipe, - ) - - await node.waitUntilStarted() - - var execResult = await execCommandEx( - "cd " & node.workingDir() & " && " & node.workingDir() / node.executable() & - " run " & node.workingDir() / "scripts" / "mine.js --network localhost" - ) - - assert execResult.status == 0 - - execResult = await execCommandEx( - "cd " & node.workingDir() & " && " & node.workingDir() / node.executable() & - " ignition deploy ignition/modules/marketplace.js --network localhost" - ) - - assert execResult.status == 0 - - trace "hardhat post start scripts executed" - except CancelledError as error: - raise error - except CatchableError as e: - error "failed to start hardhat process", error = e.msg - -proc startNode*( - _: type HardhatProcess, - args: seq[string], - debug: string | bool = false, - name: string, -): Future[HardhatProcess] {.async.} = - var logFilePath = "" - - var arguments = newSeq[string]() - for arg in args: - if arg.contains "--log-file=": - logFilePath = arg.split("=")[1] - else: - arguments.add arg - - trace "starting hardhat node", arguments - ## Starts a Hardhat Node with the specified arguments. - ## Set debug to 'true' to see output of the node. - let hardhat = HardhatProcess( - arguments: arguments, - debug: ($debug != "false"), - trackedFutures: TrackedFutures.new(), - name: "hardhat", - ) - - await hardhat.start() - - if logFilePath != "": - hardhat.logFile = some hardhat.openLogFile(logFilePath) - - return hardhat - -method onOutputLineCaptured(node: HardhatProcess, line: string) = - without logFile =? node.logFile: - return - - if error =? logFile.writeFile(line & "\n").errorOption: - error "failed to write to hardhat file", errorCode = $error - discard logFile.closeFile() - node.logFile = none IoHandle - -method stop*(node: HardhatProcess) {.async.} = - # terminate the process - await procCall NodeProcess(node).stop() - - if logFile =? node.logFile: - trace "closing hardhat log file" - discard logFile.closeFile() - -method removeDataDir*(node: HardhatProcess) = - discard diff --git a/tests/integration/marketplacesuite.nim b/tests/integration/marketplacesuite.nim deleted file mode 100644 index 5a0a11a6..00000000 --- a/tests/integration/marketplacesuite.nim +++ /dev/null @@ -1,149 +0,0 @@ -import pkg/chronos -import pkg/ethers/erc20 -from pkg/libp2p import Cid -import pkg/codex/contracts/marketplace as mp -import pkg/codex/periods -import pkg/codex/utils/json -from pkg/codex/utils import roundUp, divUp -import ./multinodes except Subscription -import ../contracts/time -import ../contracts/deployment - -export mp -export multinodes - -template marketplacesuite*(name: string, stopOnRequestFail: bool, body: untyped) = - multinodesuite name: - var marketplace {.inject, used.}: Marketplace - var period: uint64 - var periodicity: Periodicity - var token {.inject, used.}: Erc20Token - var requestStartedEvent: AsyncEvent - var requestStartedSubscription: Subscription - var requestFailedEvent: AsyncEvent - var requestFailedSubscription: Subscription - - proc onRequestStarted(eventResult: ?!RequestFulfilled) {.raises: [].} = - requestStartedEvent.fire() - - proc onRequestFailed(eventResult: ?!RequestFailed) {.raises: [].} = - requestFailedEvent.fire() - if stopOnRequestFail: - fail() - - proc getCurrentPeriod(): Future[Period] {.async.} = - return periodicity.periodOf((await ethProvider.currentTime()).truncate(uint64)) - - proc waitForRequestToStart( - seconds = 10 * 60 + 10 - ): Future[Period] {.async: (raises: [CancelledError, AsyncTimeoutError]).} = - await requestStartedEvent.wait().wait(timeout = chronos.seconds(seconds)) - # Recreate a new future if we need to wait for another request - requestStartedEvent = newAsyncEvent() - - proc waitForRequestToFail( - seconds = (5 * 60) + 10 - ): Future[Period] {.async: (raises: [CancelledError, AsyncTimeoutError]).} = - await requestFailedEvent.wait().wait(timeout = chronos.seconds(seconds)) - # Recreate a new future if we need to wait for another request - requestFailedEvent = newAsyncEvent() - - proc advanceToNextPeriod() {.async.} = - let periodicity = Periodicity(seconds: period) - let currentTime = (await ethProvider.currentTime()).truncate(uint64) - let currentPeriod = periodicity.periodOf(currentTime) - let endOfPeriod = periodicity.periodEnd(currentPeriod) - await ethProvider.advanceTimeTo(endOfPeriod.u256 + 1) - - template eventuallyP(condition: untyped, finalPeriod: Period): bool = - proc eventuallyP(): Future[bool] {.async: (raises: [CancelledError]).} = - while ( - let currentPeriod = await getCurrentPeriod() - currentPeriod <= finalPeriod - ) - : - if condition: - return true - await sleepAsync(1.millis) - return condition - - await eventuallyP() - - proc periods(p: int): uint64 = - p.uint64 * period - - proc slotSize(blocks, nodes, tolerance: int): UInt256 = - let ecK = nodes - tolerance - let blocksRounded = roundUp(blocks, ecK) - let blocksPerSlot = divUp(blocksRounded, ecK) - (DefaultBlockSize * blocksPerSlot.NBytes).Natural.u256 - - proc datasetSize(blocks, nodes, tolerance: int): UInt256 = - return nodes.u256 * slotSize(blocks, nodes, tolerance) - - proc createAvailabilities( - datasetSize: uint64, - duration: uint64, - collateralPerByte: UInt256, - minPricePerBytePerSecond: UInt256, - ): Future[void] {.async: (raises: [CancelledError, HttpError, ConfigurationError]).} = - let totalCollateral = datasetSize.u256 * collateralPerByte - # post availability to each provider - for i in 0 ..< providers().len: - let provider = providers()[i].client - - discard await provider.postAvailability( - totalSize = datasetSize, - duration = duration.uint64, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - ) - - proc requestStorage( - client: CodexClient, - cid: Cid, - proofProbability = 1.u256, - duration: uint64 = 12.periods, - pricePerBytePerSecond = 1.u256, - collateralPerByte = 1.u256, - expiry: uint64 = 4.periods, - nodes = providers().len, - tolerance = 0, - ): Future[PurchaseId] {.async: (raises: [CancelledError, HttpError]).} = - let id = ( - await client.requestStorage( - cid, - expiry = expiry, - duration = duration, - proofProbability = proofProbability, - collateralPerByte = collateralPerByte, - pricePerBytePerSecond = pricePerBytePerSecond, - nodes = nodes.uint, - tolerance = tolerance.uint, - ) - ).get - - return id - - setup: - marketplace = Marketplace.new(Marketplace.address, ethProvider.getSigner()) - let tokenAddress = await marketplace.token() - token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) - let config = await marketplace.configuration() - period = config.proofs.period - periodicity = Periodicity(seconds: period) - - requestStartedEvent = newAsyncEvent() - requestFailedEvent = newAsyncEvent() - - requestStartedSubscription = - await marketplace.subscribe(RequestFulfilled, onRequestStarted) - - requestFailedSubscription = - await marketplace.subscribe(RequestFailed, onRequestFailed) - - teardown: - await requestStartedSubscription.unsubscribe() - await requestFailedSubscription.unsubscribe() - - body diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 5f149585..4ed94ca8 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -6,21 +6,15 @@ import std/times import pkg/codex/conf import pkg/codex/logutils import pkg/chronos/transports/stream -import pkg/ethers import pkg/questionable import ./codexconfig import ./codexprocess -import ./hardhatconfig -import ./hardhatprocess import ./nodeconfigs import ../asynctest import ../checktest export asynctest -export ethers except `%` -export hardhatprocess export codexprocess -export hardhatconfig export codexconfig export nodeconfigs @@ -31,9 +25,6 @@ type Role* {.pure.} = enum Client - Provider - Validator - Hardhat MultiNodeSuiteError = object of CatchableError @@ -73,33 +64,11 @@ proc getTempDirName*(starttime: string, role: Role, roleIdx: int): string = template multinodesuite*(name: string, body: untyped) = asyncchecksuite name: - # Following the problem described here: - # https://github.com/NomicFoundation/hardhat/issues/2053 - # It may be desirable to use http RPC provider. - # This turns out to be equally important in tests where - # subscriptions get wiped out after 5mins even when - # a new block is mined. - # For this reason, we are using http provider here as the default. - # To use a different provider in your test, you may use - # multinodesuiteWithProviderUrl template in your tests. - # If you want to use a different provider url in the nodes, you can - # use withEthProvider config modifier in the node config - # to set the desired provider url. E.g.: - # NodeConfigs( - # hardhat: - # HardhatConfig.none, - # clients: - # CodexConfigs.init(nodes=1) - # .withEthProvider("ws://localhost:8545") - # .some, - # ... var running {.inject, used.}: seq[RunningNode] var bootstrapNodes: seq[string] let starttime = now().format("yyyy-MM-dd'_'HH:mm:ss") var currentTestName = "" var nodeConfigs: NodeConfigs - var ethProvider {.inject, used.}: JsonRpcProvider - var accounts {.inject, used.}: seq[Address] var snapshot: JsonNode template test(tname, startNodeConfigs, tbody) = @@ -131,31 +100,11 @@ template multinodesuite*(name: string, body: untyped) = let fileName = logDir / fn return fileName - proc newHardhatProcess( - config: HardhatConfig, role: Role - ): Future[NodeProcess] {.async.} = - var args: seq[string] = @[] - if config.logFile: - let updatedLogFile = getLogFile(role, none int) - args.add "--log-file=" & updatedLogFile - - try: - let node = await HardhatProcess.startNode(args, config.debugEnabled, "hardhat") - trace "hardhat node started" - return node - except NodeProcessError as e: - raiseMultiNodeSuiteError "cannot start hardhat process: " & e.msg - proc newCodexProcess( roleIdx: int, conf: CodexConfig, role: Role ): Future[NodeProcess] {.async.} = let nodeIdx = running.len var config = conf - - if nodeIdx > accounts.len - 1: - raiseMultiNodeSuiteError "Cannot start node at nodeIdx " & $nodeIdx & - ", not enough eth accounts." - let datadir = getTempDirName(starttime, role, roleIdx) try: @@ -185,91 +134,22 @@ template multinodesuite*(name: string, body: untyped) = return node - proc hardhat(): HardhatProcess = - for r in running: - if r.role == Role.Hardhat: - return HardhatProcess(r.node) - return nil - proc clients(): seq[CodexProcess] {.used.} = return collect: for r in running: if r.role == Role.Client: CodexProcess(r.node) - proc providers(): seq[CodexProcess] {.used.} = - return collect: - for r in running: - if r.role == Role.Provider: - CodexProcess(r.node) - - proc validators(): seq[CodexProcess] {.used.} = - return collect: - for r in running: - if r.role == Role.Validator: - CodexProcess(r.node) - - proc startHardhatNode(config: HardhatConfig): Future[NodeProcess] {.async.} = - return await newHardhatProcess(config, Role.Hardhat) - proc startClientNode(conf: CodexConfig): Future[NodeProcess] {.async.} = let clientIdx = clients().len - var config = conf - config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) - config.addCliOption( - StartUpCmd.persistence, "--eth-account", $accounts[running.len] - ) - return await newCodexProcess(clientIdx, config, Role.Client) - - proc startProviderNode(conf: CodexConfig): Future[NodeProcess] {.async.} = - let providerIdx = providers().len - var config = conf - config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) - config.addCliOption( - StartUpCmd.persistence, "--eth-account", $accounts[running.len] - ) - config.addCliOption( - PersistenceCmd.prover, "--circom-r1cs", - "vendor/logos-storage-contracts-eth/verifier/networks/hardhat/proof_main.r1cs", - ) - config.addCliOption( - PersistenceCmd.prover, "--circom-wasm", - "vendor/logos-storage-contracts-eth/verifier/networks/hardhat/proof_main.wasm", - ) - config.addCliOption( - PersistenceCmd.prover, "--circom-zkey", - "vendor/logos-storage-contracts-eth/verifier/networks/hardhat/proof_main.zkey", - ) - - return await newCodexProcess(providerIdx, config, Role.Provider) - - proc startValidatorNode(conf: CodexConfig): Future[NodeProcess] {.async.} = - let validatorIdx = validators().len - var config = conf - config.addCliOption(StartUpCmd.persistence, "--eth-provider", jsonRpcProviderUrl) - config.addCliOption( - StartUpCmd.persistence, "--eth-account", $accounts[running.len] - ) - config.addCliOption(StartUpCmd.persistence, "--validator") - - return await newCodexProcess(validatorIdx, config, Role.Validator) + return await newCodexProcess(clientIdx, conf, Role.Client) proc teardownImpl() {.async.} = - for nodes in @[validators(), clients(), providers()]: + for nodes in @[clients()]: for node in nodes: await node.stop() # also stops rest client node.removeDataDir() - # if hardhat was started in the test, kill the node - # otherwise revert the snapshot taken in the test setup - let hardhat = hardhat() - if not hardhat.isNil: - await hardhat.stop() - else: - discard await send(ethProvider, "evm_revert", @[snapshot]) - - await ethProvider.close() - running = @[] template failAndTeardownOnError(message: string, tryBody: untyped) = @@ -294,31 +174,6 @@ template multinodesuite*(name: string, body: untyped) = bootstrapNodes.add ninfo["spr"].getStr() setup: - if var conf =? nodeConfigs.hardhat: - try: - let node = await startHardhatNode(conf) - running.add RunningNode(role: Role.Hardhat, node: node) - except CatchableError as e: - echo "failed to start hardhat node" - fail() - quit(1) - - try: - # Workaround for https://github.com/NomicFoundation/hardhat/issues/2053 - # Do not use websockets, but use http and polling to stop subscriptions - # from being removed after 5 minutes - ethProvider = JsonRpcProvider.new(jsonRpcProviderUrl) - # if hardhat was NOT started by the test, take a snapshot so it can be - # reverted in the test teardown - if nodeConfigs.hardhat.isNone: - snapshot = await send(ethProvider, "evm_snapshot") - accounts = await ethProvider.listAccounts() - except CatchableError as e: - echo "Hardhat not running. Run hardhat manually " & - "before executing tests, or include a " & "HardhatConfig in the test setup." - fail() - quit(1) - if var clients =? nodeConfigs.clients: failAndTeardownOnError "failed to start client nodes": for config in clients.configs: @@ -326,22 +181,6 @@ template multinodesuite*(name: string, body: untyped) = running.add RunningNode(role: Role.Client, node: node) await CodexProcess(node).updateBootstrapNodes() - if var providers =? nodeConfigs.providers: - failAndTeardownOnError "failed to start provider nodes": - for config in providers.configs.mitems: - let node = await startProviderNode(config) - running.add RunningNode(role: Role.Provider, node: node) - await CodexProcess(node).updateBootstrapNodes() - - if var validators =? nodeConfigs.validators: - failAndTeardownOnError "failed to start validator nodes": - for config in validators.configs.mitems: - let node = await startValidatorNode(config) - running.add RunningNode(role: Role.Validator, node: node) - - # ensure that we have a recent block with a fresh timestamp - discard await send(ethProvider, "evm_mine") - teardown: await teardownImpl() diff --git a/tests/integration/nodeconfigs.nim b/tests/integration/nodeconfigs.nim index 19e797e3..5f11bf55 100644 --- a/tests/integration/nodeconfigs.nim +++ b/tests/integration/nodeconfigs.nim @@ -1,9 +1,5 @@ import pkg/questionable import ./codexconfig -import ./hardhatconfig type NodeConfigs* = object clients*: ?CodexConfigs - providers*: ?CodexConfigs - validators*: ?CodexConfigs - hardhat*: ?HardhatConfig diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index eeceb20d..a49fddcb 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -18,12 +18,8 @@ template twonodessuite*(name: string, body: untyped) = var node2 {.inject, used.}: CodexProcess var client1 {.inject, used.}: CodexClient var client2 {.inject, used.}: CodexClient - var account1 {.inject, used.}: Address - var account2 {.inject, used.}: Address setup: - account1 = accounts[0] - account2 = accounts[1] node1 = clients()[0] node2 = clients()[1] diff --git a/tests/testContracts.nim b/tests/testContracts.nim deleted file mode 100644 index 4b0b7d93..00000000 --- a/tests/testContracts.nim +++ /dev/null @@ -1,6 +0,0 @@ -import std/os -import ./imports - -importTests(currentSourcePath().parentDir() / "contracts") - -{.warning[UnusedImport]: off.} diff --git a/tests/testTaiko.nim b/tests/testTaiko.nim deleted file mode 100644 index b1555bfb..00000000 --- a/tests/testTaiko.nim +++ /dev/null @@ -1,82 +0,0 @@ -import std/times -import std/os -import std/json -import std/tempfiles -import pkg/chronos -import pkg/stint -import pkg/questionable -import pkg/questionable/results - -import ./asynctest -import ./integration/nodes - -suite "Taiko L2 Integration Tests": - var node1, node2: NodeProcess - - setup: - doAssert existsEnv("CODEX_ETH_PRIVATE_KEY"), "Key for Taiko account missing" - - node1 = startNode( - [ - "--data-dir=" & createTempDir("", ""), "--api-port=8080", "--nat=none", - "--disc-port=8090", "--persistence", "--eth-provider=https://rpc.test.taiko.xyz", - ] - ) - node1.waitUntilStarted() - - let bootstrap = (!(await node1.client.info()))["spr"].getStr() - - node2 = startNode( - [ - "--data-dir=" & createTempDir("", ""), - "--api-port=8081", - "--nat=none", - "--disc-port=8091", - "--bootstrap-node=" & bootstrap, - "--persistence", - "--eth-provider=https://rpc.test.taiko.xyz", - ] - ) - node2.waitUntilStarted() - - teardown: - node1.stop() - node2.stop() - node1.removeDataDir() - node2.removeDataDir() - - test "node 1 buys storage from node 2": - let size = 0xFFFFF.u256 - let minPricePerBytePerSecond = 1.u256 - let totalCollateral = size * minPricePerBytePerSecond - discard node2.client.postAvailability( - size = size, - duration = 200.u256, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - ) - let cid = !node1.client.upload("some file contents") - - echo " - requesting storage, expires in 5 minutes" - let expiry = getTime().toUnix().uint64 + 5 * 60 - let purchase = - !node1.client.requestStorage( - cid, - duration = 30.u256, - pricePerBytePerSecond = 1.u256, - proofProbability = 3.u256, - collateralPerByte = 1.u256, - expiry = expiry.u256, - ) - - echo " - waiting for request to start, timeout 5 minutes" - check eventually( - node1.client.getPurchase(purchase) .? state == success "started", - timeout = 5 * 60 * 1000, - ) - - echo " - waiting for request to finish, timeout 1 minute" - check eventually( - node1.client.getPurchase(purchase) .? state == success "finished", - timeout = 1 * 60 * 1000, - ) diff --git a/tests/testTools.nim b/tests/testTools.nim deleted file mode 100644 index dbeb04b0..00000000 --- a/tests/testTools.nim +++ /dev/null @@ -1,6 +0,0 @@ -import std/os -import ./imports - -importTests(currentSourcePath().parentDir() / "tools") - -{.warning[UnusedImport]: off.} diff --git a/tests/tools/cirdl/testcirdl.nim b/tests/tools/cirdl/testcirdl.nim deleted file mode 100644 index 8311a456..00000000 --- a/tests/tools/cirdl/testcirdl.nim +++ /dev/null @@ -1,35 +0,0 @@ -import std/os -import std/osproc -import std/options -import pkg/chronos -import pkg/codex/contracts -import ../../asynctest -import ../../contracts/deployment - -suite "tools/cirdl": - const - cirdl = "build" / "cirdl" - workdir = "." - - test "circuit download tool": - let - circuitPath = "testcircuitpath" - rpcEndpoint = "ws://localhost:8545" - marketplaceAddress = Marketplace.address - - discard existsOrCreateDir(circuitPath) - - let args = [circuitPath, rpcEndpoint, $marketplaceAddress] - - let process = osproc.startProcess(cirdl, workdir, args, options = {poParentStreams}) - - let returnCode = process.waitForExit() - check returnCode == 0 - - check: - fileExists(circuitPath / "proof_main_verification_key.json") - fileExists(circuitPath / "proof_main.r1cs") - fileExists(circuitPath / "proof_main.wasm") - fileExists(circuitPath / "proof_main.zkey") - - removeDir(circuitPath) diff --git a/tools/cirdl/cirdl.nim b/tools/cirdl/cirdl.nim deleted file mode 100644 index 5b363fc3..00000000 --- a/tools/cirdl/cirdl.nim +++ /dev/null @@ -1,152 +0,0 @@ -import std/os -import std/streams -import pkg/chronicles -import pkg/chronos -import pkg/ethers -import pkg/questionable -import pkg/questionable/results -import pkg/zippy/tarballs -import pkg/chronos/apps/http/httpclient -import ../../codex/contracts/marketplace -import ../../codex/contracts/deployment - -proc consoleLog(logLevel: LogLevel, msg: LogOutputStr) {.gcsafe.} = - try: - stdout.write(msg) - stdout.flushFile() - except IOError as err: - logLoggingFailure(cstring(msg), err) - -proc noOutput(logLevel: LogLevel, msg: LogOutputStr) = - discard - -defaultChroniclesStream.outputs[0].writer = consoleLog -defaultChroniclesStream.outputs[1].writer = noOutput -defaultChroniclesStream.outputs[2].writer = noOutput - -proc printHelp() = - info "Usage: ./cirdl [circuitPath] [rpcEndpoint] ([marketplaceAddress])" - info " circuitPath: path where circuit files will be placed." - info " rpcEndpoint: URL of web3 RPC endpoint." - info " marketplaceAddress: Address of deployed Storage marketplace contracts. If left out, will auto-discover based on connected network." - -proc getMarketplaceAddress( - provider: JsonRpcProvider, mpAddressOverride: ?Address -): Future[?Address] {.async.} = - let deployment = Deployment.new(provider, mpAddressOverride) - let address = await deployment.address(Marketplace) - - return address - -proc getCircuitHash( - provider: JsonRpcProvider, marketplaceAddress: Address -): Future[?!string] {.async.} = - let marketplace = Marketplace.new(marketplaceAddress, provider) - let config = await marketplace.configuration() - return success config.proofs.zkeyHash - -proc formatUrl(hash: string): string = - "https://circuit.codex.storage/proving-key/" & hash - -proc retrieveUrl(uri: string): Future[seq[byte]] {.async.} = - let httpSession = HttpSessionRef.new() - try: - let resp = await httpSession.fetch(parseUri(uri)) - return resp.data - finally: - await noCancel(httpSession.closeWait()) - -proc downloadZipfile(url: string, filepath: string): Future[?!void] {.async.} = - try: - let file = await retrieveUrl(url) - var s = newFileStream(filepath, fmWrite) - for b in file: - s.write(b) - s.close() - except Exception as exc: - return failure(exc.msg) - success() - -proc unzip(zipfile: string, targetPath: string): ?!void = - try: - extractAll(zipfile, targetPath) - except Exception as exc: - return failure(exc.msg) - success() - -proc copyFiles(unpackDir: string, circuitPath: string): ?!void = - try: - for file in walkDir(unpackDir): - copyFileToDir(file.path, circuitPath) - except Exception as exc: - return failure(exc.msg) - success() - -proc main() {.async.} = - info "Storage Circuit Downloader, Aww yeah!" - let args = os.commandLineParams() - if args.len < 2 or args.len > 3: - printHelp() - return - - let - circuitPath = args[0] - rpcEndpoint = args[1] - zipfile = "circuit.tar.gz" - unpackFolder = "." / "tempunpackfolder" - - var mpAddressOverride: ?Address - - if args.len == 3: - without parsed =? Address.init(args[2]): - raise newException(ValueError, "Invalid ethereum address") - mpAddressOverride = some parsed - - debug "Starting", circuitPath, rpcEndpoint - - if (dirExists(unpackFolder)): - removeDir(unpackFolder) - - let provider = JsonRpcProvider.new(rpcEndpoint) - - without marketplaceAddress =? - (await getMarketplaceAddress(provider, mpAddressOverride)), err: - error "No known marketplace address, nor any specified manually", msg = err.msg - return - - info "Marketplace address", address = $marketplaceAddress - - without circuitHash =? (await getCircuitHash(provider, marketplaceAddress)), err: - error "Failed to get circuit hash", msg = err.msg - return - debug "Got circuithash", circuitHash - - let url = formatUrl(circuitHash) - info "Download URL", url - if dlErr =? (await downloadZipfile(url, zipfile)).errorOption: - error "Failed to download circuit file", msg = dlErr.msg - return - debug "Download completed" - - if err =? unzip(zipfile, unpackFolder).errorOption: - error "Failed to unzip file", msg = err.msg - return - debug "Unzip completed" - - # Unpack library cannot unpack into existing directory. We also cannot - # delete the targer directory and have the library recreate it because - # Logos Storage has likely created it and set correct permissions. - # So, we unpack to a temp folder and move the files. - if err =? copyFiles(unpackFolder, circuitPath).errorOption: - error "Failed to copy files", msg = err.msg - return - debug "Files copied" - - removeFile(zipfile) - removeDir(unpackFolder) - - debug "file and unpack folder removed" - -when isMainModule: - waitFor main() - info "Done!" diff --git a/vendor/urls.rules b/vendor/urls.rules deleted file mode 100644 index 7636ff34..00000000 --- a/vendor/urls.rules +++ /dev/null @@ -1,8 +0,0 @@ -https://github.com/status-im/nim-libp2p-dht.git -> https://github.com/codex-storage/nim-codex-dht.git -https://github.com/markspanbroek/questionable -> https://github.com/codex-storage/questionable -https://github.com/status-im/questionable -> https://github.com/codex-storage/questionable -https://github.com/status-im/asynctest -> https://github.com/codex-storage/asynctest -https://github.com/status-im/nim-datastore -> https://github.com/codex-storage/nim-datastore -https://github.com/cheatfate/nimcrypto -> https://github.com/status-im/nimcrypto -protobufserialization -> protobuf_serialization -protobufserialization -> https://github.com/status-im/nim-protobuf-serialization