From fe3d9bc97704cc8545e148956f30aa818879e160 Mon Sep 17 00:00:00 2001 From: Arnaud Date: Mon, 27 Jan 2025 17:01:15 +0100 Subject: [PATCH] Add slotCollateral calculation with getRequest cache and remove populationItem function --- codex/codex.nim | 4 +- codex/conf.nim | 9 ++++ codex/contracts/market.nim | 32 +++++++++---- codex/market.nim | 4 +- codex/sales.nim | 73 +++++++++++++---------------- codex/sales/slotqueue.nim | 18 ------- codex/sales/states/filling.nim | 11 ++--- tests/codex/helpers/mockmarket.nim | 19 ++++++-- tests/codex/sales/testsales.nim | 20 ++++++-- tests/codex/sales/testslotqueue.nim | 22 --------- tests/contracts/testMarket.nim | 47 +++++++++++++++++-- 11 files changed, 144 insertions(+), 115 deletions(-) diff --git a/codex/codex.nim b/codex/codex.nim index 13985254..8a49180b 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -107,7 +107,9 @@ proc bootstrapInteractions(s: CodexServer): Future[void] {.async.} = quit QuitFailure let marketplace = Marketplace.new(marketplaceAddress, signer) - let market = OnChainMarket.new(marketplace, config.rewardRecipient) + let market = OnChainMarket.new( + marketplace, config.rewardRecipient, config.marketplaceRequestCacheSize + ) let clock = OnChainClock.new(provider) var client: ?ClientInteractions diff --git a/codex/conf.nim b/codex/conf.nim index 6d47f8f4..79bebb48 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -347,6 +347,15 @@ type name: "reward-recipient" .}: Option[EthAddress] + marketplaceRequestCacheSize* {. + desc: + "The size of the request cache - " & + "reduce the contract calls to get the request data.", + defaultValue: 128, + defaultValueDesc: "128", + name: "request-cache-size" + .}: uint16 + case persistenceCmd* {.defaultValue: noCmd, command.}: PersistenceCmd of PersistenceCmd.prover: circuitDir* {. diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index e73bdfeb..9194efcd 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -2,6 +2,7 @@ import std/strutils import pkg/ethers import pkg/upraises import pkg/questionable +import pkg/lrucache import ../utils/exceptions import ../logutils import ../market @@ -20,6 +21,7 @@ type signer: Signer rewardRecipient: ?Address configuration: ?MarketplaceConfig + requestCache: LruCache[string, StorageRequest] MarketSubscription = market.Subscription EventSubscription = ethers.Subscription @@ -27,12 +29,22 @@ type eventSubscription: EventSubscription func new*( - _: type OnChainMarket, contract: Marketplace, rewardRecipient = Address.none + _: type OnChainMarket, + contract: Marketplace, + rewardRecipient = Address.none, + requestCacheSize: uint16 = 0, ): OnChainMarket = without signer =? contract.signer: raiseAssert("Marketplace contract should have a signer") - OnChainMarket(contract: contract, signer: signer, rewardRecipient: rewardRecipient) + var requestCache = newLruCache[string, StorageRequest](int(requestCacheSize)) + + OnChainMarket( + contract: contract, + signer: signer, + rewardRecipient: rewardRecipient, + requestCache: requestCache, + ) proc raiseMarketError(message: string) {.raises: [MarketError].} = raise newException(MarketError, message) @@ -112,9 +124,16 @@ method requestStorage(market: OnChainMarket, request: StorageRequest) {.async.} method getRequest*( market: OnChainMarket, id: RequestId ): Future[?StorageRequest] {.async.} = + let key = $id + + if market.requestCache.contains(key): + return some market.requestCache[key] + convertEthersError: try: - return some await market.contract.getRequest(id) + let request = await market.contract.getRequest(id) + market.requestCache[key] = request + return some request except Marketplace_UnknownRequest: return none StorageRequest @@ -477,10 +496,3 @@ method queryPastStorageRequestedEvents*( let fromBlock = await market.contract.provider.pastBlockTag(blocksAgo) return await market.queryPastStorageRequestedEvents(fromBlock) - -method calculateRepairCollateral*( - market: Market, collateral: UInt256 -): Future[UInt256] {.async.} = - convertEthersError: - let repairRewardPercentage = (await market.repairRewardPercentage).u256 - return collateral - (collateral * repairRewardPercentage).div(100.u256) diff --git a/codex/market.nim b/codex/market.nim index 0fbae8bf..8904ff0b 100644 --- a/codex/market.nim +++ b/codex/market.nim @@ -264,7 +264,7 @@ method queryPastStorageRequestedEvents*( ): Future[seq[StorageRequested]] {.base, async.} = raiseAssert("not implemented") -method calculateRepairCollateral*( - market: Market, collateral: UInt256 +method slotCollateral*( + market: Market, requestId: RequestId, slotIndex: UInt256 ): Future[UInt256] {.base, async.} = raiseAssert("not implemented") diff --git a/codex/sales.nim b/codex/sales.nim index 0c19d4ef..6b1c5437 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -152,20 +152,15 @@ proc cleanUp( # Re-add items back into the queue to prevent small availabilities from # draining the queue. Seen items will be ordered last. if reprocessSlot and request =? data.request: - let slotId = slotId(data.requestId, data.slotIndex) - let slotState = await sales.context.market.slotState(slotId) - let collateral = - if slotState == SlotState.Repair: - await sales.context.market.calculateRepairCollateral(data.ask.collateral) - else: - data.ask.collateral + let slotCollateral = + await sales.context.market.slotCollateral(data.requestId, data.slotIndex) let queue = sales.context.slotQueue var seenItem = SlotQueueItem.init( data.requestId, data.slotIndex.truncate(uint16), StorageAsk( - collateral: collateral, + collateral: slotCollateral, duration: request.ask.duration, reward: request.ask.reward, slotSize: request.ask.slotSize, @@ -332,43 +327,39 @@ proc onSlotFreed(sales: Sales, requestId: RequestId, slotIndex: UInt256) = let context = sales.context let market = context.market let queue = context.slotQueue + var slotQueueItem: SlotQueueItem - # first attempt to populate request using existing slot metadata in queue - without var found =? queue.populateItem(requestId, slotIndex.truncate(uint16)): - trace "no existing request metadata, getting request info from contract" - # if there's no existing slot for that request, retrieve the request - # from the contract. - try: - without request =? await market.getRequest(requestId): - error "unknown request in contract" - return + try: + without request =? await market.getRequest(requestId): + error "unknown request in contract" + return - # Take the repairing state into consideration to calculate the collateral. - # This is particularly needed because it will affect the priority in the queue - # and we want to give the user the ability to tweak the parameters. - # Adding the repairing state directly in the queue priority calculation - # would not allow this flexibility. - let collateral = await market.calculateRepairCollateral(request.ask.collateral) + # Take the repairing state into consideration to calculate the collateral. + # This is particularly needed because it will affect the priority in the queue + # and we want to give the user the ability to tweak the parameters. + # Adding the repairing state directly in the queue priority calculation + # would not allow this flexibility. + let slotCollateral = await market.slotCollateral(request.id, slotIndex) - found = SlotQueueItem.init( - request.id, - slotIndex.truncate(uint16), - StorageAsk( - collateral: collateral, - duration: request.ask.duration, - reward: request.ask.reward, - slotSize: request.ask.slotSize, - ), - request.expiry, - ) - except CancelledError: - discard # do not propagate as addSlotToQueue was asyncSpawned - except CatchableError as e: - error "failed to get request from contract and add slots to queue", - error = e.msgDetail + slotQueueItem = SlotQueueItem.init( + request.id, + slotIndex.truncate(uint16), + StorageAsk( + collateral: slotCollateral, + duration: request.ask.duration, + reward: request.ask.reward, + slotSize: request.ask.slotSize, + ), + request.expiry, + ) - if err =? queue.push(found).errorOption: - error "failed to push slot items to queue", error = err.msgDetail + if err =? queue.push(slotQueueItem).errorOption: + error "failed to push slot items to queue", error = err.msgDetail + except CancelledError: + discard # do not propagate as addSlotToQueue was asyncSpawned + except CatchableError as e: + error "failed to get request from contract and add slots to queue", + error = e.msgDetail let fut = addSlotToQueue() sales.trackedFutures.track(fut) diff --git a/codex/sales/slotqueue.nim b/codex/sales/slotqueue.nim index 80ca0827..cc7b4ccd 100644 --- a/codex/sales/slotqueue.nim +++ b/codex/sales/slotqueue.nim @@ -234,24 +234,6 @@ proc unpause*(self: SlotQueue) = # set unpaused flag to true -- unblocks coroutines waiting on unpaused.wait() self.unpaused.fire() -proc populateItem*( - self: SlotQueue, requestId: RequestId, slotIndex: uint16 -): ?SlotQueueItem = - trace "populate item, items in queue", len = self.queue.len - for item in self.queue.items: - trace "populate item search", itemRequestId = item.requestId, requestId - if item.requestId == requestId: - return some SlotQueueItem( - requestId: requestId, - slotIndex: slotIndex, - slotSize: item.slotSize, - duration: item.duration, - reward: item.reward, - collateral: item.collateral, - expiry: item.expiry, - ) - return none SlotQueueItem - proc push*(self: SlotQueue, item: SlotQueueItem): ?!void = logScope: requestId = item.requestId diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index 98419f53..4e624dff 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -28,6 +28,7 @@ method onFailed*(state: SaleFilling, request: StorageRequest): ?State = method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} = let data = SalesAgent(machine).data let market = SalesAgent(machine).context.market + without (fullCollateral =? data.request .? ask .? collateral): raiseAssert "Request not set" @@ -35,17 +36,11 @@ method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} = requestId = data.requestId slotIndex = data.slotIndex - let slotId = slotId(data.requestId, data.slotIndex) - let slotState = await market.slotState(slotId) - let collateral = - if slotState == SlotState.Repair: - await market.calculateRepairCollateral(fullCollateral) - else: - fullCollateral + let slotCollateral = await market.slotCollateral(data.requestId, data.slotIndex) debug "Filling slot" try: - await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) + await market.fillSlot(data.requestId, data.slotIndex, state.proof, slotCollateral) except MarketError as e: if e.msg.contains "Slot is not free": debug "Slot is already filled, ignoring slot" diff --git a/tests/codex/helpers/mockmarket.nim b/tests/codex/helpers/mockmarket.nim index 213e3e88..b75ef89c 100644 --- a/tests/codex/helpers/mockmarket.nim +++ b/tests/codex/helpers/mockmarket.nim @@ -516,8 +516,19 @@ method unsubscribe*(subscription: ProofSubmittedSubscription) {.async.} = method unsubscribe*(subscription: SlotReservationsFullSubscription) {.async.} = subscription.market.subscriptions.onSlotReservationsFull.keepItIf(it != subscription) -method calculateRepairCollateral*( - market: MockMarket, collateral: UInt256 +method slotCollateral*( + market: MockMarket, requestId: RequestId, slotIndex: UInt256 ): Future[UInt256] {.async.} = - let repairRewardPercentage = market.config.collateral.repairRewardPercentage.u256 - return collateral - ((collateral * repairRewardPercentage)).div(100.u256) + without request =? await market.getRequest(requestId): + raise newException(MarketError, "Cannot retrieve the request.") + + let slotid = slotId(requestId, slotIndex) + let s: SlotState = await slotState(market, slotid) + + if s == SlotState.Repair: + let repairRewardPercentage = market.config.collateral.repairRewardPercentage.u256 + return + request.ask.collateral - + (request.ask.collateral * repairRewardPercentage).div(100.u256) + + return request.ask.collateral diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim index d8838857..76eb8755 100644 --- a/tests/codex/sales/testsales.nim +++ b/tests/codex/sales/testsales.nim @@ -290,9 +290,23 @@ asyncchecksuite "Sales": createAvailability() market.requested.add request # "contract" must be able to return request + market.emitSlotFreed(request.id, 2.u256) - let expected = SlotQueueItem.init(request, 2.uint16) + let slotCollateral = await market.slotCollateral(request.id, 2.u256) + + let expected = SlotQueueItem.init( + request.id, + 2.uint16, + StorageAsk( + collateral: slotCollateral, + duration: request.ask.duration, + reward: request.ask.reward, + slotSize: request.ask.slotSize, + ), + request.expiry, + ) + check eventually itemsProcessed.contains(expected) test "items in queue are readded (and marked seen) once ignored": @@ -612,7 +626,3 @@ asyncchecksuite "Sales": await sales.load() check (await reservations.all(Reservation)).get.len == 0 check getAvailability().freeSize == availability.freeSize # was restored - - test "calculates correctly the collateral when the slot is being repaired": - let collateral = await market.calculateRepairCollateral(collateral = 100.u256) - check collateral == 90.u256 diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim index e6583bb7..c4768931 100644 --- a/tests/codex/sales/testslotqueue.nim +++ b/tests/codex/sales/testslotqueue.nim @@ -313,28 +313,6 @@ suite "Slot queue": check isOk queue.push(item3) check isOk queue.push(item4) - test "populates item with exisiting request metadata": - newSlotQueue(maxSize = 8, maxWorkers = 1, processSlotDelay = 10.millis) - let request0 = StorageRequest.example - var request1 = StorageRequest.example - request1.ask.collateral += 1.u256 - let items0 = SlotQueueItem.init(request0) - let items1 = SlotQueueItem.init(request1) - check queue.push(items0).isOk - check queue.push(items1).isOk - let populated = !queue.populateItem(request1.id, 12'u16) - check populated.requestId == request1.id - check populated.slotIndex == 12'u16 - check populated.slotSize == request1.ask.slotSize - check populated.duration == request1.ask.duration - check populated.reward == request1.ask.reward - check populated.collateral == request1.ask.collateral - - test "does not find exisiting request metadata": - newSlotQueue(maxSize = 2, maxWorkers = 2) - let item = SlotQueueItem.example - check queue.populateItem(item.requestId, 12'u16).isNone - test "can support uint16.high slots": var request = StorageRequest.example let maxUInt16 = uint16.high diff --git a/tests/contracts/testMarket.nim b/tests/contracts/testMarket.nim index 5e6a632a..d71ecc50 100644 --- a/tests/contracts/testMarket.nim +++ b/tests/contracts/testMarket.nim @@ -2,6 +2,7 @@ import std/options import std/importutils import pkg/chronos import pkg/ethers/erc20 +import pkg/lrucache import codex/contracts import ../ethertest import ./examples @@ -18,6 +19,7 @@ logScope: ethersuite "On-Chain Market": let proof = Groth16Proof.example + let requestCacheSize = 128.uint16 var market: OnChainMarket var marketplace: Marketplace @@ -37,7 +39,7 @@ ethersuite "On-Chain Market": proc switchAccount(account: Signer) = marketplace = marketplace.connect(account) token = token.connect(account) - market = OnChainMarket.new(marketplace, market.rewardRecipient) + market = OnChainMarket.new(marketplace, market.rewardRecipient, requestCacheSize) setup: let address = Marketplace.address(dummyVerifier = true) @@ -45,7 +47,7 @@ ethersuite "On-Chain Market": let config = await marketplace.configuration() hostRewardRecipient = accounts[2] - market = OnChainMarket.new(marketplace) + market = OnChainMarket.new(marketplace, requestCacheSize = requestCacheSize) let tokenAddress = await marketplace.token() token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) @@ -83,7 +85,8 @@ ethersuite "On-Chain Market": test "fails to instantiate when contract does not have a signer": let storageWithoutSigner = marketplace.connect(ethProvider) expect AssertionDefect: - discard OnChainMarket.new(storageWithoutSigner) + discard + OnChainMarket.new(storageWithoutSigner, requestCacheSize = requestCacheSize) test "knows signer address": check (await market.getSigner()) == (await ethProvider.getSigner().getAddress()) @@ -549,7 +552,9 @@ ethersuite "On-Chain Market": check endBalance == (startBalance + expectedPayout + request.ask.collateral) test "pays rewards to reward recipient, collateral to host": - market = OnChainMarket.new(marketplace, hostRewardRecipient.some) + market = OnChainMarket.new( + marketplace, hostRewardRecipient.some, requestCacheSize = requestCacheSize + ) let hostAddress = await host.getAddress() await market.requestStorage(request) @@ -577,3 +582,37 @@ ethersuite "On-Chain Market": let expectedPayout = request.expectedPayout(filledAt, requestEnd.u256) check endBalanceHost == (startBalanceHost + request.ask.collateral) check endBalanceReward == (startBalanceReward + expectedPayout) + + test "the request is added in cache after the fist access": + await market.requestStorage(request) + + check market.requestCache.contains($request.id) == false + discard await market.getRequest(request.id) + + check market.requestCache.contains($request.id) == true + let cacheValue = market.requestCache[$request.id] + check cacheValue == request + + test "returns the collateral when the slot is not being repaired": + await market.requestStorage(request) + await market.reserveSlot(request.id, 0.u256) + await market.fillSlot(request.id, 0.u256, proof, request.ask.collateral) + + let slotId = request.slotId(0.u256) + let collateral = await market.slotCollateral(request.id, 0.u256) + + check collateral == request.ask.collateral + + test "calculates correctly the collateral when the slot is being repaired": + await market.requestStorage(request) + await market.reserveSlot(request.id, 0.u256) + await market.fillSlot(request.id, 0.u256, proof, request.ask.collateral) + await market.freeSlot(slotId(request.id, 0.u256)) + + let slotId = request.slotId(0.u256) + let collateral = await market.slotCollateral(request.id, 0.u256) + + # slotCollateral = 200 + # repairRewardPercentage = 10 + # expected collateral = 200 - 200 * 0.1 = 180 + check collateral == 180