diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 2aac641c..3a054933 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -49,6 +49,32 @@ logScope: declareCounter(codex_api_uploads, "codex API uploads") declareCounter(codex_api_downloads, "codex API downloads") +proc getLongestRequestEnd( + node: CodexNodeRef, availabilityId: AvailabilityId +): Future[?!SecondsSince1970] {.async: (raises: []).} = + without contracts =? node.contracts.host: + return failure("Sales unavailable") + + let + reservations = contracts.sales.context.reservations + market = contracts.sales.context.market + + try: + without allReservations =? await reservations.all(Reservation, availabilityId): + return failure("Cannot retrieve the reservations") + + echo "all reservations is done" + let requestEnds = allReservations.mapIt(await market.getRequestEnd(it.requestId)) + + if len(requestEnds) == 0: + return success(0.SecondsSince1970) + + return success(requestEnds.max) + except CancelledError, CatchableError: + error "Error when trying to get longest request end", + error = getCurrentExceptionMsg() + return failure("Cannot retrieve the request dates") + proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} = 0 @@ -433,6 +459,14 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = Http400, "Total size must be larger then zero", headers = headers ) + let until = restAv.until |? 0 + if until < 0: + return RestApiResponse.error( + Http400, + "Until parameter must be greater or equal 0. Got: " & $until, + headers = headers, + ) + if not reservations.hasAvailable(restAv.totalSize.truncate(uint)): return RestApiResponse.error(Http422, "Not enough storage quota", headers = headers) @@ -444,6 +478,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = restAv.minPricePerBytePerSecond, restAv.totalCollateral, enabled = restAv.enabled |? true, + until = until, ) ), error: return RestApiResponse.error(Http500, error.msg, headers = headers) @@ -529,7 +564,24 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) = if totalCollateral =? restAv.totalCollateral: availability.totalCollateral = totalCollateral - availability.enabled = restAv.enabled |? true + if until =? restAv.until: + if until < 0: + return RestApiResponse.error( + Http400, "Until parameter must be greater or equal 0. Got: " & $until + ) + + without longestRequestEnd =? (await node.getLongestRequestEnd(id)).catch, err: + return RestApiResponse.error(Http500, err.msg) + + if until > 0 and until < longestRequestEnd.get: + return RestApiResponse.error( + Http400, + "Until parameter must be greater or equal the current longest request.", + ) + + availability.until = until + + availability.enabled = restAv.enabled if err =? (await reservations.update(availability)).errorOption: return RestApiResponse.error(Http500, err.msg) diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 14124cd0..d88f183f 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -34,6 +34,7 @@ type totalCollateral* {.serialize.}: UInt256 freeSize* {.serialize.}: ?UInt256 enabled* {.serialize.}: ?bool + until* {.serialize.}: ?SecondsSince1970 RestSalesAgent* = object state* {.serialize.}: string diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index fca83dc8..70f42a5f 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -35,6 +35,7 @@ import std/sequtils import std/sugar import std/typetraits import std/sequtils +import std/times import pkg/chronos import pkg/datastore import pkg/nimcrypto @@ -73,7 +74,11 @@ type # If false, the availability will not be able to receive new slots. # If it is turned on and the availability is already hosting slots, # it will not affect those existing slots. - enabled* {.serialize.}: bool + # It should be implicitly set to true when creating + # a new availability. Therefore, it is not required when updating an existing availability. + enabled* {.serialize.}: ?bool + # 0 means non-restricted, otherwise contains timestamp until the Availability will be renewed + until* {.serialize.}: SecondsSince1970 Reservation* = ref object id* {.serialize.}: ReservationId @@ -133,7 +138,8 @@ proc init*( duration: UInt256, minPricePerBytePerSecond: UInt256, totalCollateral: UInt256, - enabled: bool, + enabled: ?bool, + until: SecondsSince1970, ): Availability = var id: array[32, byte] doAssert randomBytes(id) == 32 @@ -146,6 +152,7 @@ proc init*( totalCollateral: totalCollateral, totalRemainingCollateral: totalCollateral, enabled: enabled, + until: until, ) func totalCollateral*(self: Availability): UInt256 {.inline.} = @@ -275,7 +282,9 @@ proc updateAvailability( trace "Creating new Availability" let res = await self.updateImpl(obj) # inform subscribers that Availability has been added - if obj.enabled and onAvailabilityAdded =? self.onAvailabilityAdded: + without var enabled =? obj.enabled: + enabled = oldAvailability.enabled |? true + if enabled and onAvailabilityAdded =? self.onAvailabilityAdded: await onAvailabilityAdded(obj) return res else: @@ -301,7 +310,9 @@ proc updateAvailability( let res = await self.updateImpl(obj) - if obj.enabled and oldAvailability.freeSize < obj.freeSize: # availability added + without var enabled =? obj.enabled: + enabled = oldAvailability.enabled |? true + if enabled and oldAvailability.freeSize < obj.freeSize: # availability added # inform subscribers that Availability has been modified (with increased # size) if onAvailabilityAdded =? self.onAvailabilityAdded: @@ -380,12 +391,13 @@ proc createAvailability*( minPricePerBytePerSecond: UInt256, totalCollateral: UInt256, enabled: bool, + until: SecondsSince1970, ): Future[?!Availability] {.async.} = trace "creating availability", - size, duration, minPricePerBytePerSecond, totalCollateral, enabled + size, duration, minPricePerBytePerSecond, totalCollateral, enabled, until let availability = Availability.init( - size, size, duration, minPricePerBytePerSecond, totalCollateral, enabled + size, size, duration, minPricePerBytePerSecond, totalCollateral, enabled.some, until ) let bytes = availability.freeSize.truncate(uint) @@ -637,15 +649,17 @@ proc findAvailability*( error "failed to get all storables", error = e.msg return none Availability + let endTime = getTime().toUnix() + cast[int64](duration) for item in storables.items: if bytes =? (await item) and availability =? Availability.fromJson(bytes): - if availability.enabled and size <= availability.freeSize and - duration <= availability.duration and + let enabled = availability.enabled |? true + if enabled and size <= availability.freeSize and duration <= availability.duration and collateralPerByte <= availability.maxCollateralPerByte and - pricePerBytePerSecond >= availability.minPricePerBytePerSecond: + pricePerBytePerSecond >= availability.minPricePerBytePerSecond and + (availability.until == 0 or availability.until >= endTime): trace "availability matched", id = availability.id, - enabled = availability.enabled, + enabled = enabled, size, availFreeSize = availability.freeSize, duration, diff --git a/tests/codex/examples.nim b/tests/codex/examples.nim index 5ea99956..38f6712f 100644 --- a/tests/codex/examples.nim +++ b/tests/codex/examples.nim @@ -65,7 +65,8 @@ proc example*( duration = uint16.example.u256, minPricePerBytePerSecond = uint8.example.u256, totalCollateral = totalSize * collateralPerByte, - enabled = true, + enabled = some true, + until = 0.SecondsSince1970, ) proc example*(_: type Reservation): Reservation = diff --git a/tests/codex/sales/states/testpreparing.nim b/tests/codex/sales/states/testpreparing.nim index 97c8ea0b..166e41ff 100644 --- a/tests/codex/sales/states/testpreparing.nim +++ b/tests/codex/sales/states/testpreparing.nim @@ -39,7 +39,8 @@ asyncchecksuite "sales state 'preparing'": duration = request.ask.duration + 60.u256, minPricePerBytePerSecond = request.ask.pricePerBytePerSecond, totalCollateral = request.ask.collateralPerSlot * request.ask.slots.u256, - enabled = true, + enabled = some true, + until = 0.SecondsSince1970, ) let repoDs = SQLiteDatastore.new(Memory).tryGet() let metaDs = SQLiteDatastore.new(Memory).tryGet() @@ -70,8 +71,12 @@ asyncchecksuite "sales state 'preparing'": proc createAvailability(enabled = true) {.async.} = let a = await reservations.createAvailability( - availability.totalSize, availability.duration, - availability.minPricePerBytePerSecond, availability.totalCollateral, enabled, + availability.totalSize, + availability.duration, + availability.minPricePerBytePerSecond, + availability.totalCollateral, + enabled, + until = 0.SecondsSince1970, ) availability = a.get diff --git a/tests/codex/sales/testreservations.nim b/tests/codex/sales/testreservations.nim index d8880579..2048ba43 100644 --- a/tests/codex/sales/testreservations.nim +++ b/tests/codex/sales/testreservations.nim @@ -1,5 +1,5 @@ import std/random - +import std/times import pkg/questionable import pkg/questionable/results import pkg/chronos @@ -8,6 +8,7 @@ import pkg/datastore import pkg/codex/stores import pkg/codex/errors import pkg/codex/sales +import pkg/codex/clock import pkg/codex/utils/json import ../../asynctest @@ -39,13 +40,13 @@ asyncchecksuite "Reservations module": await repoTmp.destroyDb() await metaTmp.destroyDb() - proc createAvailability(enabled = true): Availability = + proc createAvailability(enabled = true, until = 0.SecondsSince1970): Availability = let example = Availability.example(collateralPerByte) let totalSize = rand(100000 .. 200000).u256 let totalCollateral = totalSize * collateralPerByte let availability = waitFor reservations.createAvailability( totalSize, example.duration, example.minPricePerBytePerSecond, totalCollateral, - enabled, + enabled, until, ) return availability.get @@ -65,8 +66,12 @@ asyncchecksuite "Reservations module": check (await reservations.all(Availability)).get.len == 0 test "generates unique ids for storage availability": - let availability1 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256, true) - let availability2 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256, true) + let availability1 = Availability.init( + 1.u256, 2.u256, 3.u256, 4.u256, 5.u256, some true, 0.SecondsSince1970 + ) + let availability2 = Availability.init( + 1.u256, 2.u256, 3.u256, 4.u256, 5.u256, some true, 0.SecondsSince1970 + ) check availability1.id != availability2.id test "can reserve available storage": @@ -260,6 +265,21 @@ asyncchecksuite "Reservations module": check isOk await reservations.update(availability) check (repo.quotaReservedBytes - origQuota) == 100.NBytes + test "enabled is updated correctly": + let availability = createAvailability() + + check availability.enabled.get == true + + check isOk await reservations.update(availability) + let key = availability.key.get + var updated = !(await reservations.get(key, Availability)) + check updated.enabled.get == true + + availability.enabled = false.some + check isOk await reservations.update(availability) + updated = !(await reservations.get(key, Availability)) + check updated.enabled.get == false + test "reservation can be partially released": let availability = createAvailability() let reservation = createReservation(availability) @@ -352,6 +372,31 @@ asyncchecksuite "Reservations module": check found.isNone + test "finds an availability when the until date is after the duration": + let example = Availability.example(collateralPerByte) + let until = getTime().toUnix() + cast[int64](example.duration) + let availability = createAvailability(until = until) + + let found = await reservations.findAvailability( + availability.freeSize, availability.duration, + availability.minPricePerBytePerSecond, collateralPerByte, + ) + + check found.isSome + check found.get == availability + + test "does not find an availability when the until date is before the duration": + let example = Availability.example(collateralPerByte) + let until = getTime().toUnix() + 1.SecondsSince1970 + let availability = createAvailability(until = until) + + let found = await reservations.findAvailability( + availability.freeSize, availability.duration, + availability.minPricePerBytePerSecond, collateralPerByte, + ) + + check found.isNone + test "non-matching availabilities are not found": let availability = createAvailability() @@ -394,6 +439,7 @@ asyncchecksuite "Reservations module": UInt256.example, UInt256.example, enabled = true, + until = 0.SecondsSince1970, ) check created.isErr check created.error of ReserveFailedError diff --git a/tests/codex/sales/testsales.nim b/tests/codex/sales/testsales.nim index 02c8addb..ff8056a4 100644 --- a/tests/codex/sales/testsales.nim +++ b/tests/codex/sales/testsales.nim @@ -14,6 +14,7 @@ import pkg/codex/stores/repostore import pkg/codex/blocktype as bt import pkg/codex/node import pkg/codex/utils/asyncstatemachine +import times import ../../asynctest import ../helpers import ../helpers/mockmarket @@ -149,7 +150,8 @@ asyncchecksuite "Sales": duration = 60.u256, minPricePerBytePerSecond = minPricePerBytePerSecond, totalCollateral = totalCollateral, - enabled = true, + enabled = some true, + until = 0.SecondsSince1970, ) request = StorageRequest( ask: StorageAsk( @@ -217,10 +219,11 @@ asyncchecksuite "Sales": let key = availability.id.key.get (waitFor reservations.get(key, Availability)).get - proc createAvailability(enabled = true) = + proc createAvailability(enabled = true, until = 0.SecondsSince1970) = let a = waitFor reservations.createAvailability( availability.totalSize, availability.duration, availability.minPricePerBytePerSecond, availability.totalCollateral, enabled, + until, ) availability = a.get # update id @@ -422,11 +425,32 @@ asyncchecksuite "Sales": createAvailability(enabled = true) await market.requestStorage(request) - availability.enabled = false + availability.enabled = some false discard await reservations.update(availability) check wasIgnored() + test "ignores request when availability until terminates before the duration": + let until = getTime().toUnix() + createAvailability(until = until) + await market.requestStorage(request) + + check wasIgnored() + + test "retrieves request when availability until terminates after the duration": + let until = getTime().toUnix() + cast[int64](request.ask.duration) + createAvailability(until = until) + + var storingRequest: StorageRequest + sales.onStore = proc( + request: StorageRequest, slot: UInt256, onBatch: BatchProc + ): Future[?!void] {.async.} = + storingRequest = request + return success() + + await market.requestStorage(request) + check eventually storingRequest == request + test "retrieves and stores data locally": var storingRequest: StorageRequest var storingSlot: UInt256 diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index fbaa8dcb..10b1dbcf 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -172,11 +172,12 @@ proc getSlots*(client: CodexClient): ?!seq[Slot] = let body = client.http.getContent(url) seq[Slot].fromJson(body) -proc postAvailability*( +proc postAvailabilityRaw*( client: CodexClient, totalSize, duration, minPricePerBytePerSecond, totalCollateral: UInt256, enabled: ?bool = bool.none, -): ?!Availability = + until: ?SecondsSince1970 = SecondsSince1970.none, +): Response = ## Post sales availability endpoint ## let url = client.baseurl & "/sales/availability" @@ -187,8 +188,24 @@ proc postAvailability*( "minPricePerBytePerSecond": minPricePerBytePerSecond, "totalCollateral": totalCollateral, "enabled": enabled |? true, + "until": until, } - let response = client.http.post(url, $json) + return client.http.post(url, $json) + +proc postAvailability*( + client: CodexClient, + totalSize, duration, minPricePerBytePerSecond, totalCollateral: UInt256, + enabled: ?bool = bool.none, + until: ?SecondsSince1970 = SecondsSince1970.none, +): ?!Availability = + let response = client.postAvailabilityRaw( + totalSize = totalSize, + duration = duration, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = totalCollateral, + enabled = enabled, + until = until, + ) doAssert response.status == "201 Created", "expected 201 Created, got " & response.status & ", body: " & response.body Availability.fromJson(response.body) @@ -199,6 +216,7 @@ proc patchAvailabilityRaw*( totalSize, freeSize, duration, minPricePerBytePerSecond, totalCollateral: ?UInt256 = UInt256.none, enabled: ?bool = bool.none, + until: ?SecondsSince1970 = SecondsSince1970.none, ): Response = ## Updates availability ## @@ -225,6 +243,9 @@ proc patchAvailabilityRaw*( if enabled =? enabled: json["enabled"] = %enabled + if until =? until: + json["until"] = %until + client.http.patch(url, $json) proc patchAvailability*( @@ -233,6 +254,7 @@ proc patchAvailability*( totalSize, duration, minPricePerBytePerSecond, totalCollateral: ?UInt256 = UInt256.none, enabled: ?bool = bool.none, + until: ?SecondsSince1970 = SecondsSince1970.none, ): void = let response = client.patchAvailabilityRaw( availabilityId, @@ -241,6 +263,7 @@ proc patchAvailability*( minPricePerBytePerSecond = minPricePerBytePerSecond, totalCollateral = totalCollateral, enabled = enabled, + until = until, ) doAssert response.status == "200 OK", "expected 200 OK, got " & response.status diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index bc030a1d..9705426c 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -1,3 +1,5 @@ +import std/times +import std/httpclient import ../examples import ../contracts/time import ../contracts/deployment @@ -123,6 +125,53 @@ marketplacesuite "Marketplace": timeout = 10 * 1000, # give client a bit of time to withdraw its funds ) + test "returns an error when trying to update the until date before an existing a request is finished", + marketplaceConfig: + let size = 0xFFFFFF.u256 + let data = await RandomChunker.example(blocks = blocks) + let marketplace = Marketplace.new(Marketplace.address, ethProvider.getSigner()) + let tokenAddress = await marketplace.token() + let token = Erc20Token.new(tokenAddress, ethProvider.getSigner()) + let duration = 20 * 60.u256 + + # host makes storage available + let startBalanceHost = await token.balanceOf(hostAccount) + let availability = host.postAvailability( + totalSize = size, + duration = 20 * 60.u256, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = size * minPricePerBytePerSecond, + ).get + + # client requests storage + let cid = client.upload(data).get + let id = client.requestStorage( + cid, + duration = duration, + pricePerBytePerSecond = minPricePerBytePerSecond, + proofProbability = 3.u256, + expiry = 10 * 60, + collateralPerByte = collateralPerByte, + nodes = ecNodes, + tolerance = ecTolerance, + ).get + + check eventually(client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000) + let purchase = client.getPurchase(id).get + check purchase.error == none string + + let unixNow = getTime().toUnix() + let until = unixNow + 1 + + let response = host.patchAvailabilityRaw( + availabilityId = availability.id, until = cast[SecondsSince1970](until).some + ) + + check: + response.status == "400 Bad Request" + response.body == + "Until parameter must be greater or equal the current longest request." + marketplacesuite "Marketplace payouts": const minPricePerBytePerSecond = 1.u256 const collateralPerByte = 1.u256 diff --git a/tests/integration/testrestapi.nim b/tests/integration/testrestapi.nim index 1019e7f5..1a7e7fcc 100644 --- a/tests/integration/testrestapi.nim +++ b/tests/integration/testrestapi.nim @@ -38,19 +38,6 @@ twonodessuite "REST API": space.quotaUsedBytes == 65598.NBytes space.quotaReservedBytes == 12.NBytes - test "created correctly an availability when not enabled by default", twoNodesConfig: - let totalSize = 12.u256 - let minPricePerBytePerSecond = 1.u256 - let totalCollateral = totalSize * minPricePerBytePerSecond - let availability = client1.postAvailability( - totalSize = totalSize, - duration = 2.u256, - minPricePerBytePerSecond = minPricePerBytePerSecond, - totalCollateral = totalCollateral, - enabled = false.some, - ).get - check availability.enabled == false - test "node lists local files", twoNodesConfig: let content1 = "some file contents" let content2 = "some other contents" diff --git a/tests/integration/testsales.nim b/tests/integration/testsales.nim index 6d1f76d4..b2c2ffa9 100644 --- a/tests/integration/testsales.nim +++ b/tests/integration/testsales.nim @@ -1,4 +1,5 @@ import std/httpclient +import std/times import pkg/codex/contracts import ./twonodes import ../codex/examples @@ -53,6 +54,51 @@ multinodesuite "Sales": ).get check availability in host.getAvailabilities().get + test "created correctly an availability when not enabled by default", salesConfig: + let totalSize = 12.u256 + let minPricePerBytePerSecond = 1.u256 + let totalCollateral = totalSize * minPricePerBytePerSecond + let availability = host.postAvailability( + totalSize = totalSize, + duration = 2.u256, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = totalCollateral, + enabled = false.some, + ).get + check availability.enabled == false.some + + test "create availability fails when until is negative", salesConfig: + let totalSize = 12.u256 + let minPricePerBytePerSecond = 1.u256 + let totalCollateral = totalSize * minPricePerBytePerSecond + let response = host.postAvailabilityRaw( + totalSize = totalSize, + duration = 2.u256, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = totalCollateral, + until = cast[SecondsSince1970](-1).some, + ) + + check: + response.status == "400 Bad Request" + response.body == "Until parameter must be greater or equal 0. Got: -1" + + test "create availability fails when until is negative", salesConfig: + let totalSize = 12.u256 + let minPricePerBytePerSecond = 1.u256 + let totalCollateral = totalSize * minPricePerBytePerSecond + let response = host.postAvailabilityRaw( + totalSize = totalSize, + duration = 2.u256, + minPricePerBytePerSecond = minPricePerBytePerSecond, + totalCollateral = totalCollateral, + until = cast[SecondsSince1970](-1).some, + ) + + check: + response.status == "400 Bad Request" + response.body == "Until parameter must be greater or equal 0. Got: -1" + test "updating non-existing availability", salesConfig: let nonExistingResponse = host.patchAvailabilityRaw( AvailabilityId.example, @@ -118,7 +164,21 @@ multinodesuite "Sales": ).get host.patchAvailability(availability.id, enabled = false.some) let updatedAvailability = (host.getAvailabilities().get).findItem(availability).get - check updatedAvailability.enabled == false + check updatedAvailability.enabled == false.some + + test "updating availability - updating until", salesConfig: + var until = cast[SecondsSince1970](getTime().toUnix()) + let availability = host.postAvailability( + totalSize = 140000.u256, + duration = 200.u256, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + until = until.some, + ).get + until += 10.SecondsSince1970 + host.patchAvailability(availability.id, until = until.some) + let updatedAvailability = (host.getAvailabilities().get).findItem(availability).get + check updatedAvailability.until == until test "updating availability - updating totalSize does not allow bellow utilized", salesConfig: @@ -163,3 +223,19 @@ multinodesuite "Sales": (host.getAvailabilities().get).findItem(availability).get check newUpdatedAvailability.totalSize == originalSize + 20000 check newUpdatedAvailability.freeSize - updatedAvailability.freeSize == 20000 + + test "updating availability fails with until negative", salesConfig: + let availability = host.postAvailability( + totalSize = 140000.u256, + duration = 200.u256, + minPricePerBytePerSecond = 3.u256, + totalCollateral = 300.u256, + ).get + + let response = host.patchAvailabilityRaw( + availability.id, until = cast[SecondsSince1970](-1).some + ) + + check: + response.status == "400 Bad Request" + response.body == "Until parameter must be greater or equal 0. Got: -1"