mirror of
https://github.com/status-im/nim-dagger.git
synced 2025-01-30 00:17:45 +00:00
e6a387e8e8
* add seen flag * Add MockSlotQueueItem and better prioritisation tests * Update seen priority, and include in SlotQueueItem.init * Re-add processed slots to queue Re-add processed slots to queue if the sale was ignored or errored * add pausing of queue - when processing slots in queue, pause queue if item was marked seen - if availability size is increased, trigger onAvailabilityAdded callback - in sales, on availability added, clear 'seen' flags, then unpause the queue - when items pushed to the queue, unpause the queue * remove unused NoMatchingAvailabilityError from slotqueue The slot queue should also have nothing to do with availabilities * when all availabilities are empty, pause the queue An empty availability is defined as size < DefaultBlockSize as this means even the smallest possible request could not be served. However, this is up for discussion. * remove availability from onAvailabilitiesEmptied callback * refactor onAvailabilityAdded and onAvailabilitiesEmptied onAvailabilityAdded and onAvailabilitiesEmptied are now only called from reservations.update (and eventually reservations.delete once implemented). - Add empty routine for Availability and Reservation - Add allEmpty routine for Availability and Reservation, which returns true when all all Availability or Reservation objects in the datastore are empty. * SlotQueue test support updates * Sales module test support updates * Reservations module tests for queue pausing * Sales module tests for queue pausing Includes tests for sales states cancelled, errored, ignored to ensure onCleanUp is called with correct parameters * SlotQueue module tests for queue pausing * fix existing sales test * PR feedback - indent `self.unpause` - update comment for `clearSeenFlags` * reprocessSlot in SaleErrored only when coming from downloading * remove pausing of queue when availabilities are "emptied" Queue pausing when all availiabilies are "emptied" is not necessary, given that the node would not be able to service slots once all its availabilities' freeSize are too small for the slots in the queue, and would then be paused anyway. Add test that asserts the queue is paused once the freeSpace of availabilities drops too low to fill slots in the queue. * Update clearing of seen flags The asyncheapqueue update overload would need to check index bounds and ultimately a different solution was found using the mitems iterator. * fix test request.id was different before updating request.ask.slots, and that id was used to set the state in mockmarket. * Change filled/cleanup future to nil, so no await is needed * add wait to allow items to be added to queue * do not unpause queue when seen items are pushed * re-add seen item back to queue once paused Previously, when a seen item was processed, it was first popped off the queue, then the queue was paused waiting to process that item once the queue was unpaused. Now, when a seen item is processed, it is popped off the queue, the queue is paused, then the item is re-added to the queue and the queue will wait until unpaused before it will continue popping items off the queue. If the item was not re-added to the queue, it would have been processed immediately once unpaused, however there may have been other items with higher priority pushed to the queue in the meantime. The queue would not be unpaused if those added items were already seen. In particular, this may happen when ignored items due to lack of availability are re-added to a paused queue. Those ignored items will likely have a higher priority than the item that was just seen (due to it having been processed first), causing the queue to the be paused. * address PR comments
347 lines
11 KiB
Nim
347 lines
11 KiB
Nim
import std/random
|
|
|
|
import pkg/questionable
|
|
import pkg/questionable/results
|
|
import pkg/chronos
|
|
import pkg/datastore
|
|
|
|
import pkg/codex/stores
|
|
import pkg/codex/sales
|
|
import pkg/codex/utils/json
|
|
|
|
import ../../asynctest
|
|
import ../examples
|
|
import ../helpers
|
|
|
|
asyncchecksuite "Reservations module":
|
|
var
|
|
repo: RepoStore
|
|
repoDs: Datastore
|
|
metaDs: SQLiteDatastore
|
|
reservations: Reservations
|
|
|
|
setup:
|
|
randomize(1.int64) # create reproducible results
|
|
repoDs = SQLiteDatastore.new(Memory).tryGet()
|
|
metaDs = SQLiteDatastore.new(Memory).tryGet()
|
|
repo = RepoStore.new(repoDs, metaDs)
|
|
reservations = Reservations.new(repo)
|
|
|
|
proc createAvailability(): Availability =
|
|
let example = Availability.example
|
|
let totalSize = rand(100000..200000)
|
|
let availability = waitFor reservations.createAvailability(
|
|
totalSize.u256,
|
|
example.duration,
|
|
example.minPrice,
|
|
example.maxCollateral
|
|
)
|
|
return availability.get
|
|
|
|
proc createReservation(availability: Availability): Reservation =
|
|
let size = rand(1..<availability.freeSize.truncate(int))
|
|
let reservation = waitFor reservations.createReservation(
|
|
availability.id,
|
|
size.u256,
|
|
RequestId.example,
|
|
UInt256.example
|
|
)
|
|
return reservation.get
|
|
|
|
test "availability can be serialised and deserialised":
|
|
let availability = Availability.example
|
|
let serialised = %availability
|
|
check Availability.fromJson(serialised).get == availability
|
|
|
|
test "has no availability initially":
|
|
check (await reservations.all(Availability)).get.len == 0
|
|
|
|
test "generates unique ids for storage availability":
|
|
let availability1 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256)
|
|
let availability2 = Availability.init(1.u256, 2.u256, 3.u256, 4.u256, 5.u256)
|
|
check availability1.id != availability2.id
|
|
|
|
test "can reserve available storage":
|
|
let availability = createAvailability()
|
|
check availability.id != AvailabilityId.default
|
|
|
|
test "creating availability reserves bytes in repo":
|
|
let orig = repo.available
|
|
let availability = createAvailability()
|
|
check repo.available == (orig.u256 - availability.freeSize).truncate(uint)
|
|
|
|
test "can get all availabilities":
|
|
let availability1 = createAvailability()
|
|
let availability2 = createAvailability()
|
|
let availabilities = !(await reservations.all(Availability))
|
|
check:
|
|
# perform unordered checks
|
|
availabilities.len == 2
|
|
availabilities.contains(availability1)
|
|
availabilities.contains(availability2)
|
|
|
|
test "reserved availability exists":
|
|
let availability = createAvailability()
|
|
|
|
let exists = await reservations.exists(availability.key.get)
|
|
|
|
check exists
|
|
|
|
test "reservation can be created":
|
|
let availability = createAvailability()
|
|
let reservation = createReservation(availability)
|
|
check reservation.id != ReservationId.default
|
|
|
|
test "can get all reservations":
|
|
let availability1 = createAvailability()
|
|
let availability2 = createAvailability()
|
|
let reservation1 = createReservation(availability1)
|
|
let reservation2 = createReservation(availability2)
|
|
let availabilities = !(await reservations.all(Availability))
|
|
let reservations = !(await reservations.all(Reservation))
|
|
check:
|
|
# perform unordered checks
|
|
availabilities.len == 2
|
|
reservations.len == 2
|
|
reservations.contains(reservation1)
|
|
reservations.contains(reservation2)
|
|
|
|
test "can get reservations of specific availability":
|
|
let availability1 = createAvailability()
|
|
let availability2 = createAvailability()
|
|
let reservation1 = createReservation(availability1)
|
|
let reservation2 = createReservation(availability2)
|
|
let reservations = !(await reservations.all(Reservation, availability1.id))
|
|
|
|
check:
|
|
# perform unordered checks
|
|
reservations.len == 1
|
|
reservations.contains(reservation1)
|
|
not reservations.contains(reservation2)
|
|
|
|
test "cannot create reservation with non-existant availability":
|
|
let availability = Availability.example
|
|
let created = await reservations.createReservation(
|
|
availability.id,
|
|
UInt256.example,
|
|
RequestId.example,
|
|
UInt256.example
|
|
)
|
|
check created.isErr
|
|
check created.error of NotExistsError
|
|
|
|
test "cannot create reservation larger than availability size":
|
|
let availability = createAvailability()
|
|
let created = await reservations.createReservation(
|
|
availability.id,
|
|
availability.totalSize + 1,
|
|
RequestId.example,
|
|
UInt256.example
|
|
)
|
|
check created.isErr
|
|
check created.error of BytesOutOfBoundsError
|
|
|
|
test "creating reservation reduces availability size":
|
|
let availability = createAvailability()
|
|
let orig = availability.freeSize
|
|
let reservation = createReservation(availability)
|
|
let key = availability.id.key.get
|
|
let updated = (await reservations.get(key, Availability)).get
|
|
check updated.freeSize == orig - reservation.size
|
|
|
|
test "can check if reservation exists":
|
|
let availability = createAvailability()
|
|
let reservation = createReservation(availability)
|
|
let key = reservation.key.get
|
|
check await reservations.exists(key)
|
|
|
|
test "non-existant availability does not exist":
|
|
let key = AvailabilityId.example.key.get
|
|
check not (await reservations.exists(key))
|
|
|
|
test "non-existant reservation does not exist":
|
|
let key = key(ReservationId.example, AvailabilityId.example).get
|
|
check not (await reservations.exists(key))
|
|
|
|
test "can check if availability exists":
|
|
let availability = createAvailability()
|
|
let key = availability.key.get
|
|
check await reservations.exists(key)
|
|
|
|
test "can delete reservation":
|
|
let availability = createAvailability()
|
|
let reservation = createReservation(availability)
|
|
check isOk (await reservations.deleteReservation(
|
|
reservation.id, reservation.availabilityId)
|
|
)
|
|
let key = reservation.key.get
|
|
check not (await reservations.exists(key))
|
|
|
|
test "deleting reservation returns bytes back to availability":
|
|
let availability = createAvailability()
|
|
let orig = availability.freeSize
|
|
let reservation = createReservation(availability)
|
|
discard await reservations.deleteReservation(
|
|
reservation.id, reservation.availabilityId
|
|
)
|
|
let key = availability.key.get
|
|
let updated = !(await reservations.get(key, Availability))
|
|
check updated.freeSize == orig
|
|
|
|
test "calling returnBytesToAvailability returns bytes back to availability":
|
|
let availability = createAvailability()
|
|
let reservation = createReservation(availability)
|
|
let orig = availability.freeSize - reservation.size
|
|
let origQuota = repo.quotaReservedBytes
|
|
let returnedBytes = reservation.size + 200.u256
|
|
|
|
check isOk await reservations.returnBytesToAvailability(
|
|
reservation.availabilityId, reservation.id, returnedBytes
|
|
)
|
|
|
|
let key = availability.key.get
|
|
let updated = !(await reservations.get(key, Availability))
|
|
|
|
check updated.freeSize > orig
|
|
check (updated.freeSize - orig) == 200.u256
|
|
check (repo.quotaReservedBytes - origQuota) == 200
|
|
|
|
test "update releases quota when lowering size":
|
|
let
|
|
availability = createAvailability()
|
|
origQuota = repo.quotaReservedBytes
|
|
availability.totalSize = availability.totalSize - 100
|
|
|
|
check isOk await reservations.update(availability)
|
|
check (origQuota - repo.quotaReservedBytes) == 100
|
|
|
|
test "update reserves quota when growing size":
|
|
let
|
|
availability = createAvailability()
|
|
origQuota = repo.quotaReservedBytes
|
|
availability.totalSize = availability.totalSize + 100
|
|
|
|
check isOk await reservations.update(availability)
|
|
check (repo.quotaReservedBytes - origQuota) == 100
|
|
|
|
test "reservation can be partially released":
|
|
let availability = createAvailability()
|
|
let reservation = createReservation(availability)
|
|
check isOk await reservations.release(
|
|
reservation.id,
|
|
reservation.availabilityId,
|
|
1
|
|
)
|
|
let key = reservation.key.get
|
|
let updated = !(await reservations.get(key, Reservation))
|
|
check updated.size == reservation.size - 1
|
|
|
|
test "cannot release more bytes than size of reservation":
|
|
let availability = createAvailability()
|
|
let reservation = createReservation(availability)
|
|
let updated = await reservations.release(
|
|
reservation.id,
|
|
reservation.availabilityId,
|
|
(reservation.size + 1).truncate(uint)
|
|
)
|
|
check updated.isErr
|
|
check updated.error of BytesOutOfBoundsError
|
|
|
|
test "cannot release bytes from non-existant reservation":
|
|
let availability = createAvailability()
|
|
let reservation = createReservation(availability)
|
|
let updated = await reservations.release(
|
|
ReservationId.example,
|
|
availability.id,
|
|
1
|
|
)
|
|
check updated.isErr
|
|
check updated.error of NotExistsError
|
|
|
|
test "onAvailabilityAdded called when availability is created":
|
|
var added: Availability
|
|
reservations.onAvailabilityAdded = proc(a: Availability) {.async.} =
|
|
added = a
|
|
|
|
let availability = createAvailability()
|
|
|
|
check added == availability
|
|
|
|
test "onAvailabilityAdded called when availability size is increased":
|
|
var availability = createAvailability()
|
|
var added: Availability
|
|
reservations.onAvailabilityAdded = proc(a: Availability) {.async.} =
|
|
added = a
|
|
availability.freeSize += 1.u256
|
|
discard await reservations.update(availability)
|
|
|
|
check added == availability
|
|
|
|
test "onAvailabilityAdded is not called when availability size is decreased":
|
|
var availability = createAvailability()
|
|
var called = false
|
|
reservations.onAvailabilityAdded = proc(a: Availability) {.async.} =
|
|
called = true
|
|
availability.freeSize -= 1.u256
|
|
discard await reservations.update(availability)
|
|
|
|
check not called
|
|
|
|
test "availabilities can be found":
|
|
let availability = createAvailability()
|
|
|
|
let found = await reservations.findAvailability(
|
|
availability.freeSize,
|
|
availability.duration,
|
|
availability.minPrice,
|
|
availability.maxCollateral)
|
|
|
|
check found.isSome
|
|
check found.get == availability
|
|
|
|
test "non-matching availabilities are not found":
|
|
let availability = createAvailability()
|
|
|
|
let found = await reservations.findAvailability(
|
|
availability.freeSize + 1,
|
|
availability.duration,
|
|
availability.minPrice,
|
|
availability.maxCollateral)
|
|
|
|
check found.isNone
|
|
|
|
test "non-existant availability cannot be found":
|
|
let availability = Availability.example
|
|
let found = (await reservations.findAvailability(
|
|
availability.freeSize,
|
|
availability.duration,
|
|
availability.minPrice,
|
|
availability.maxCollateral
|
|
))
|
|
check found.isNone
|
|
|
|
test "non-existant availability cannot be retrieved":
|
|
let key = AvailabilityId.example.key.get
|
|
let got = await reservations.get(key, Availability)
|
|
check got.error of NotExistsError
|
|
|
|
test "can get available bytes in repo":
|
|
check reservations.available == DefaultQuotaBytes
|
|
|
|
test "reports quota available to be reserved":
|
|
check reservations.hasAvailable(DefaultQuotaBytes - 1)
|
|
|
|
test "reports quota not available to be reserved":
|
|
check not reservations.hasAvailable(DefaultQuotaBytes + 1)
|
|
|
|
test "fails to create availability with size that is larger than available quota":
|
|
let created = await reservations.createAvailability(
|
|
(DefaultQuotaBytes + 1).u256,
|
|
UInt256.example,
|
|
UInt256.example,
|
|
UInt256.example
|
|
)
|
|
check created.isErr
|
|
check created.error of ReserveFailedError
|
|
check created.error.parent of QuotaNotEnoughError
|