status-im/nim-dagger
2
0
Fork 0
mirror of https://github.com/status-im/nim-dagger.git synced 2025-02-11 14:17:57 +00:00
Code Issues Projects Releases Wiki Activity

Add until parameter

Browse Source
This commit is contained in:
Arnaud 2025-02-05 20:52:38 +01:00
parent 7257f1768f
commit 904d20c48f
No known key found for this signature in database
GPG Key ID: 69D6CE281FCAE663
11 changed files with 318 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
codex/rest/api.nim
View File

@ -49,6 +49,32 @@ logScope:
declareCounter(codex_api_uploads, "codex API uploads")
declareCounter(codex_api_downloads, "codex API downloads")
proc getLongestRequestEnd(
node: CodexNodeRef, availabilityId: AvailabilityId
): Future[?!SecondsSince1970] {.async: (raises: []).} =
without contracts =? node.contracts.host:
return failure("Sales unavailable")
let
reservations = contracts.sales.context.reservations
market = contracts.sales.context.market
try:
without allReservations =? await reservations.all(Reservation, availabilityId):
return failure("Cannot retrieve the reservations")
echo "all reservations is done"
let requestEnds = allReservations.mapIt(await market.getRequestEnd(it.requestId))
if len(requestEnds) == 0:
return success(0.SecondsSince1970)
return success(requestEnds.max)
except CancelledError, CatchableError:
error "Error when trying to get longest request end",
error = getCurrentExceptionMsg()
return failure("Cannot retrieve the request dates")
proc validate(pattern: string, value: string): int {.gcsafe, raises: [Defect].} =
0
@ -433,6 +459,14 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
Http400, "Total size must be larger then zero", headers = headers
)
let until = restAv.until |? 0
if until < 0:
return RestApiResponse.error(
Http400,
"Until parameter must be greater or equal 0. Got: " & $until,
headers = headers,
)
if not reservations.hasAvailable(restAv.totalSize.truncate(uint)):
return
RestApiResponse.error(Http422, "Not enough storage quota", headers = headers)
@ -444,6 +478,7 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
restAv.minPricePerBytePerSecond,
restAv.totalCollateral,
enabled = restAv.enabled |? true,
until = until,
)
), error:
return RestApiResponse.error(Http500, error.msg, headers = headers)
@ -529,7 +564,24 @@ proc initSalesApi(node: CodexNodeRef, router: var RestRouter) =
if totalCollateral =? restAv.totalCollateral:
availability.totalCollateral = totalCollateral
availability.enabled = restAv.enabled |? true
if until =? restAv.until:
if until < 0:
return RestApiResponse.error(
Http400, "Until parameter must be greater or equal 0. Got: " & $until
)
without longestRequestEnd =? (await node.getLongestRequestEnd(id)).catch, err:
return RestApiResponse.error(Http500, err.msg)
if until > 0 and until < longestRequestEnd.get:
return RestApiResponse.error(
Http400,
"Until parameter must be greater or equal the current longest request.",
)
availability.until = until
availability.enabled = restAv.enabled
if err =? (await reservations.update(availability)).errorOption:
return RestApiResponse.error(Http500, err.msg)

1
codex/rest/json.nim
View File

@ -34,6 +34,7 @@ type
totalCollateral* {.serialize.}: UInt256
freeSize* {.serialize.}: ?UInt256
enabled* {.serialize.}: ?bool
until* {.serialize.}: ?SecondsSince1970
RestSalesAgent* = object
state* {.serialize.}: string

34
codex/sales/reservations.nim
View File

@ -35,6 +35,7 @@ import std/sequtils
import std/sugar
import std/typetraits
import std/sequtils
import std/times
import pkg/chronos
import pkg/datastore
import pkg/nimcrypto
@ -73,7 +74,11 @@ type
# If false, the availability will not be able to receive new slots.
# If it is turned on and the availability is already hosting slots,
# it will not affect those existing slots.
enabled* {.serialize.}: bool
# It should be implicitly set to true when creating
# a new availability. Therefore, it is not required when updating an existing availability.
enabled* {.serialize.}: ?bool
# 0 means non-restricted, otherwise contains timestamp until the Availability will be renewed
until* {.serialize.}: SecondsSince1970
Reservation* = ref object
id* {.serialize.}: ReservationId
@ -133,7 +138,8 @@ proc init*(
duration: UInt256,
minPricePerBytePerSecond: UInt256,
totalCollateral: UInt256,
enabled: bool,
enabled: ?bool,
until: SecondsSince1970,
): Availability =
var id: array[32, byte]
doAssert randomBytes(id) == 32
@ -146,6 +152,7 @@ proc init*(
totalCollateral: totalCollateral,
totalRemainingCollateral: totalCollateral,
enabled: enabled,
until: until,
)
func totalCollateral*(self: Availability): UInt256 {.inline.} =
@ -275,7 +282,9 @@ proc updateAvailability(
trace "Creating new Availability"
let res = await self.updateImpl(obj)
# inform subscribers that Availability has been added
if obj.enabled and onAvailabilityAdded =? self.onAvailabilityAdded:
without var enabled =? obj.enabled:
enabled = oldAvailability.enabled |? true
if enabled and onAvailabilityAdded =? self.onAvailabilityAdded:
await onAvailabilityAdded(obj)
return res
else:
@ -301,7 +310,9 @@ proc updateAvailability(
let res = await self.updateImpl(obj)
if obj.enabled and oldAvailability.freeSize < obj.freeSize: # availability added
without var enabled =? obj.enabled:
enabled = oldAvailability.enabled |? true
if enabled and oldAvailability.freeSize < obj.freeSize: # availability added
# inform subscribers that Availability has been modified (with increased
# size)
if onAvailabilityAdded =? self.onAvailabilityAdded:
@ -380,12 +391,13 @@ proc createAvailability*(
minPricePerBytePerSecond: UInt256,
totalCollateral: UInt256,
enabled: bool,
until: SecondsSince1970,
): Future[?!Availability] {.async.} =
trace "creating availability",
size, duration, minPricePerBytePerSecond, totalCollateral, enabled
size, duration, minPricePerBytePerSecond, totalCollateral, enabled, until
let availability = Availability.init(
size, size, duration, minPricePerBytePerSecond, totalCollateral, enabled
size, size, duration, minPricePerBytePerSecond, totalCollateral, enabled.some, until
)
let bytes = availability.freeSize.truncate(uint)
@ -637,15 +649,17 @@ proc findAvailability*(
error "failed to get all storables", error = e.msg
return none Availability
let endTime = getTime().toUnix() + cast[int64](duration)
for item in storables.items:
if bytes =? (await item) and availability =? Availability.fromJson(bytes):
if availability.enabled and size <= availability.freeSize and
duration <= availability.duration and
let enabled = availability.enabled |? true
if enabled and size <= availability.freeSize and duration <= availability.duration and
collateralPerByte <= availability.maxCollateralPerByte and
pricePerBytePerSecond >= availability.minPricePerBytePerSecond:
pricePerBytePerSecond >= availability.minPricePerBytePerSecond and
(availability.until == 0 or availability.until >= endTime):
trace "availability matched",
id = availability.id,
enabled = availability.enabled,
enabled = enabled,
size,
availFreeSize = availability.freeSize,
duration,

3
tests/codex/examples.nim
View File

@ -65,7 +65,8 @@ proc example*(
duration = uint16.example.u256,
minPricePerBytePerSecond = uint8.example.u256,
totalCollateral = totalSize * collateralPerByte,
enabled = true,
enabled = some true,
until = 0.SecondsSince1970,
)
proc example*(_: type Reservation): Reservation =

11
tests/codex/sales/states/testpreparing.nim
View File

@ -39,7 +39,8 @@ asyncchecksuite "sales state 'preparing'":
duration = request.ask.duration + 60.u256,
minPricePerBytePerSecond = request.ask.pricePerBytePerSecond,
totalCollateral = request.ask.collateralPerSlot * request.ask.slots.u256,
enabled = true,
enabled = some true,
until = 0.SecondsSince1970,
)
let repoDs = SQLiteDatastore.new(Memory).tryGet()
let metaDs = SQLiteDatastore.new(Memory).tryGet()
@ -70,8 +71,12 @@ asyncchecksuite "sales state 'preparing'":
proc createAvailability(enabled = true) {.async.} =
let a = await reservations.createAvailability(
availability.totalSize, availability.duration,
availability.minPricePerBytePerSecond, availability.totalCollateral, enabled,
availability.totalSize,
availability.duration,
availability.minPricePerBytePerSecond,
availability.totalCollateral,
enabled,
until = 0.SecondsSince1970,
)
availability = a.get

56
tests/codex/sales/testreservations.nim
View File

@ -1,5 +1,5 @@
import std/random
import std/times
import pkg/questionable
import pkg/questionable/results
import pkg/chronos
@ -8,6 +8,7 @@ import pkg/datastore
import pkg/codex/stores
import pkg/codex/errors
import pkg/codex/sales
import pkg/codex/clock
import pkg/codex/utils/json
import ../../asynctest
@ -39,13 +40,13 @@ asyncchecksuite "Reservations module":
await repoTmp.destroyDb()
await metaTmp.destroyDb()
proc createAvailability(enabled = true): Availability =
proc createAvailability(enabled = true, until = 0.SecondsSince1970): Availability =
let example = Availability.example(collateralPerByte)
let totalSize = rand(100000 .. 200000).u256
let totalCollateral = totalSize * collateralPerByte
let availability = waitFor reservations.createAvailability(
totalSize, example.duration, example.minPricePerBytePerSecond, totalCollateral,
enabled,
enabled, until,
)
return availability.get
@ -65,8 +66,12 @@ asyncchecksuite "Reservations module":
check (await reservations.all(Availability)).get.len == 0
test "generates unique ids for storage availability":
let availability1 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256, true)
let availability2 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256, true)
let availability1 = Availability.init(
1.u256, 2.u256, 3.u256, 4.u256, 5.u256, some true, 0.SecondsSince1970
)
let availability2 = Availability.init(
1.u256, 2.u256, 3.u256, 4.u256, 5.u256, some true, 0.SecondsSince1970
)
check availability1.id != availability2.id
test "can reserve available storage":
@ -260,6 +265,21 @@ asyncchecksuite "Reservations module":
check isOk await reservations.update(availability)
check (repo.quotaReservedBytes - origQuota) == 100.NBytes
test "enabled is updated correctly":
let availability = createAvailability()
check availability.enabled.get == true
check isOk await reservations.update(availability)
let key = availability.key.get
var updated = !(await reservations.get(key, Availability))
check updated.enabled.get == true
availability.enabled = false.some
check isOk await reservations.update(availability)
updated = !(await reservations.get(key, Availability))
check updated.enabled.get == false
test "reservation can be partially released":
let availability = createAvailability()
let reservation = createReservation(availability)
@ -352,6 +372,31 @@ asyncchecksuite "Reservations module":
check found.isNone
test "finds an availability when the until date is after the duration":
let example = Availability.example(collateralPerByte)
let until = getTime().toUnix() + cast[int64](example.duration)
let availability = createAvailability(until = until)
let found = await reservations.findAvailability(
availability.freeSize, availability.duration,
availability.minPricePerBytePerSecond, collateralPerByte,
)
check found.isSome
check found.get == availability
test "does not find an availability when the until date is before the duration":
let example = Availability.example(collateralPerByte)
let until = getTime().toUnix() + 1.SecondsSince1970
let availability = createAvailability(until = until)
let found = await reservations.findAvailability(
availability.freeSize, availability.duration,
availability.minPricePerBytePerSecond, collateralPerByte,
)
check found.isNone
test "non-matching availabilities are not found":
let availability = createAvailability()
@ -394,6 +439,7 @@ asyncchecksuite "Reservations module":
UInt256.example,
UInt256.example,
enabled = true,
until = 0.SecondsSince1970,
)
check created.isErr
check created.error of ReserveFailedError

30
tests/codex/sales/testsales.nim
View File

@ -14,6 +14,7 @@ import pkg/codex/stores/repostore
import pkg/codex/blocktype as bt
import pkg/codex/node
import pkg/codex/utils/asyncstatemachine
import times
import ../../asynctest
import ../helpers
import ../helpers/mockmarket
@ -149,7 +150,8 @@ asyncchecksuite "Sales":
duration = 60.u256,
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = totalCollateral,
enabled = true,
enabled = some true,
until = 0.SecondsSince1970,
)
request = StorageRequest(
ask: StorageAsk(
@ -217,10 +219,11 @@ asyncchecksuite "Sales":
let key = availability.id.key.get
(waitFor reservations.get(key, Availability)).get
proc createAvailability(enabled = true) =
proc createAvailability(enabled = true, until = 0.SecondsSince1970) =
let a = waitFor reservations.createAvailability(
availability.totalSize, availability.duration,
availability.minPricePerBytePerSecond, availability.totalCollateral, enabled,
until,
)
availability = a.get # update id
@ -422,11 +425,32 @@ asyncchecksuite "Sales":
createAvailability(enabled = true)
await market.requestStorage(request)
availability.enabled = false
availability.enabled = some false
discard await reservations.update(availability)
check wasIgnored()
test "ignores request when availability until terminates before the duration":
let until = getTime().toUnix()
createAvailability(until = until)
await market.requestStorage(request)
check wasIgnored()
test "retrieves request when availability until terminates after the duration":
let until = getTime().toUnix() + cast[int64](request.ask.duration)
createAvailability(until = until)
var storingRequest: StorageRequest
sales.onStore = proc(
request: StorageRequest, slot: UInt256, onBatch: BatchProc
): Future[?!void] {.async.} =
storingRequest = request
return success()
await market.requestStorage(request)
check eventually storingRequest == request
test "retrieves and stores data locally":
var storingRequest: StorageRequest
var storingSlot: UInt256

29
tests/integration/codexclient.nim
View File

@ -172,11 +172,12 @@ proc getSlots*(client: CodexClient): ?!seq[Slot] =
let body = client.http.getContent(url)
seq[Slot].fromJson(body)
proc postAvailability*(
proc postAvailabilityRaw*(
client: CodexClient,
totalSize, duration, minPricePerBytePerSecond, totalCollateral: UInt256,
enabled: ?bool = bool.none,
): ?!Availability =
until: ?SecondsSince1970 = SecondsSince1970.none,
): Response =
## Post sales availability endpoint
##
let url = client.baseurl & "/sales/availability"
@ -187,8 +188,24 @@ proc postAvailability*(
"minPricePerBytePerSecond": minPricePerBytePerSecond,
"totalCollateral": totalCollateral,
"enabled": enabled |? true,
"until": until,
}
let response = client.http.post(url, $json)
return client.http.post(url, $json)
proc postAvailability*(
client: CodexClient,
totalSize, duration, minPricePerBytePerSecond, totalCollateral: UInt256,
enabled: ?bool = bool.none,
until: ?SecondsSince1970 = SecondsSince1970.none,
): ?!Availability =
let response = client.postAvailabilityRaw(
totalSize = totalSize,
duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = totalCollateral,
enabled = enabled,
until = until,
)
doAssert response.status == "201 Created",
"expected 201 Created, got " & response.status & ", body: " & response.body
Availability.fromJson(response.body)
@ -199,6 +216,7 @@ proc patchAvailabilityRaw*(
totalSize, freeSize, duration, minPricePerBytePerSecond, totalCollateral: ?UInt256 =
UInt256.none,
enabled: ?bool = bool.none,
until: ?SecondsSince1970 = SecondsSince1970.none,
): Response =
## Updates availability
##
@ -225,6 +243,9 @@ proc patchAvailabilityRaw*(
if enabled =? enabled:
json["enabled"] = %enabled
if until =? until:
json["until"] = %until
client.http.patch(url, $json)
proc patchAvailability*(
@ -233,6 +254,7 @@ proc patchAvailability*(
totalSize, duration, minPricePerBytePerSecond, totalCollateral: ?UInt256 =
UInt256.none,
enabled: ?bool = bool.none,
until: ?SecondsSince1970 = SecondsSince1970.none,
): void =
let response = client.patchAvailabilityRaw(
availabilityId,
@ -241,6 +263,7 @@ proc patchAvailability*(
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = totalCollateral,
enabled = enabled,
until = until,
)
doAssert response.status == "200 OK", "expected 200 OK, got " & response.status

49
tests/integration/testmarketplace.nim
View File

@ -1,3 +1,5 @@
import std/times
import std/httpclient
import ../examples
import ../contracts/time
import ../contracts/deployment
@ -123,6 +125,53 @@ marketplacesuite "Marketplace":
timeout = 10 * 1000, # give client a bit of time to withdraw its funds
)
test "returns an error when trying to update the until date before an existing a request is finished",
marketplaceConfig:
let size = 0xFFFFFF.u256
let data = await RandomChunker.example(blocks = blocks)
let marketplace = Marketplace.new(Marketplace.address, ethProvider.getSigner())
let tokenAddress = await marketplace.token()
let token = Erc20Token.new(tokenAddress, ethProvider.getSigner())
let duration = 20 * 60.u256
# host makes storage available
let startBalanceHost = await token.balanceOf(hostAccount)
let availability = host.postAvailability(
totalSize = size,
duration = 20 * 60.u256,
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = size * minPricePerBytePerSecond,
).get
# client requests storage
let cid = client.upload(data).get
let id = client.requestStorage(
cid,
duration = duration,
pricePerBytePerSecond = minPricePerBytePerSecond,
proofProbability = 3.u256,
expiry = 10 * 60,
collateralPerByte = collateralPerByte,
nodes = ecNodes,
tolerance = ecTolerance,
).get
check eventually(client.purchaseStateIs(id, "started"), timeout = 10 * 60 * 1000)
let purchase = client.getPurchase(id).get
check purchase.error == none string
let unixNow = getTime().toUnix()
let until = unixNow + 1
let response = host.patchAvailabilityRaw(
availabilityId = availability.id, until = cast[SecondsSince1970](until).some
)
check:
response.status == "400 Bad Request"
response.body ==
"Until parameter must be greater or equal the current longest request."
marketplacesuite "Marketplace payouts":
const minPricePerBytePerSecond = 1.u256
const collateralPerByte = 1.u256

13
tests/integration/testrestapi.nim
View File

@ -38,19 +38,6 @@ twonodessuite "REST API":
space.quotaUsedBytes == 65598.NBytes
space.quotaReservedBytes == 12.NBytes
test "created correctly an availability when not enabled by default", twoNodesConfig:
let totalSize = 12.u256
let minPricePerBytePerSecond = 1.u256
let totalCollateral = totalSize * minPricePerBytePerSecond
let availability = client1.postAvailability(
totalSize = totalSize,
duration = 2.u256,
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = totalCollateral,
enabled = false.some,
).get
check availability.enabled == false
test "node lists local files", twoNodesConfig:
let content1 = "some file contents"
let content2 = "some other contents"

78
tests/integration/testsales.nim
View File

@ -1,4 +1,5 @@
import std/httpclient
import std/times
import pkg/codex/contracts
import ./twonodes
import ../codex/examples
@ -53,6 +54,51 @@ multinodesuite "Sales":
).get
check availability in host.getAvailabilities().get
test "created correctly an availability when not enabled by default", salesConfig:
let totalSize = 12.u256
let minPricePerBytePerSecond = 1.u256
let totalCollateral = totalSize * minPricePerBytePerSecond
let availability = host.postAvailability(
totalSize = totalSize,
duration = 2.u256,
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = totalCollateral,
enabled = false.some,
).get
check availability.enabled == false.some
test "create availability fails when until is negative", salesConfig:
let totalSize = 12.u256
let minPricePerBytePerSecond = 1.u256
let totalCollateral = totalSize * minPricePerBytePerSecond
let response = host.postAvailabilityRaw(
totalSize = totalSize,
duration = 2.u256,
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = totalCollateral,
until = cast[SecondsSince1970](-1).some,
)
check:
response.status == "400 Bad Request"
response.body == "Until parameter must be greater or equal 0. Got: -1"
test "create availability fails when until is negative", salesConfig:
let totalSize = 12.u256
let minPricePerBytePerSecond = 1.u256
let totalCollateral = totalSize * minPricePerBytePerSecond
let response = host.postAvailabilityRaw(
totalSize = totalSize,
duration = 2.u256,
minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = totalCollateral,
until = cast[SecondsSince1970](-1).some,
)
check:
response.status == "400 Bad Request"
response.body == "Until parameter must be greater or equal 0. Got: -1"
test "updating non-existing availability", salesConfig:
let nonExistingResponse = host.patchAvailabilityRaw(
AvailabilityId.example,
@ -118,7 +164,21 @@ multinodesuite "Sales":
).get
host.patchAvailability(availability.id, enabled = false.some)
let updatedAvailability = (host.getAvailabilities().get).findItem(availability).get
check updatedAvailability.enabled == false
check updatedAvailability.enabled == false.some
test "updating availability - updating until", salesConfig:
var until = cast[SecondsSince1970](getTime().toUnix())
let availability = host.postAvailability(
totalSize = 140000.u256,
duration = 200.u256,
minPricePerBytePerSecond = 3.u256,
totalCollateral = 300.u256,
until = until.some,
).get
until += 10.SecondsSince1970
host.patchAvailability(availability.id, until = until.some)
let updatedAvailability = (host.getAvailabilities().get).findItem(availability).get
check updatedAvailability.until == until
test "updating availability - updating totalSize does not allow bellow utilized",
salesConfig:
@ -163,3 +223,19 @@ multinodesuite "Sales":
(host.getAvailabilities().get).findItem(availability).get
check newUpdatedAvailability.totalSize == originalSize + 20000
check newUpdatedAvailability.freeSize - updatedAvailability.freeSize == 20000
test "updating availability fails with until negative", salesConfig:
let availability = host.postAvailability(
totalSize = 140000.u256,
duration = 200.u256,
minPricePerBytePerSecond = 3.u256,
totalCollateral = 300.u256,
).get
let response = host.patchAvailabilityRaw(
availability.id, until = cast[SecondsSince1970](-1).some
)
check:
response.status == "400 Bad Request"
response.body == "Until parameter must be greater or equal 0. Got: -1"