logos-storage/logos-storage-nim
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim.git synced 2026-05-16 00:09:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve integration tests

Browse Source
This commit is contained in:
Arnaud 2025-05-27 11:25:31 +02:00
parent 362799c798
commit 0b50e63d90
No known key found for this signature in database
GPG Key ID: B8FBC178F10CA7AE
1 changed files with 115 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

278
tests/integration/testslotrepair.nim
View File

@ -18,6 +18,68 @@ marketplacesuite "SP Slot Repair":
const blocks = 3 const blocks = 3
const ecNodes = 3 const ecNodes = 3
const ecTolerance = 1 const ecTolerance = 1
const size = slotSize(blocks, ecNodes, ecTolerance)
var filledSlotIds: seq[SlotId] = @[]
var freedSlotId = none SlotId
var requestId: RequestId
# Here we are keeping track of the slot filled using their ids.
proc onSlotFilled(eventResult: ?!SlotFilled) =
assert not eventResult.isErr
let event = !eventResult
if event.requestId == requestId:
let slotId = slotId(event.requestId, event.slotIndex)
filledSlotIds.add slotId
# Here we are retrieving the slot id freed.
# When the event is triggered, the slot id is removed
# from the filled slot id list.
proc onSlotFreed(eventResult: ?!SlotFreed) =
assert not eventResult.isErr
let event = !eventResult
let slotId = slotId(event.requestId, event.slotIndex)
if event.requestId == requestId:
assert slotId in filledSlotIds
filledSlotIds.del(filledSlotIds.find(slotId))
freedSlotId = some(slotId)
proc createPurchase(client: CodexClient): Future[PurchaseId] {.async.} =
let data = await RandomChunker.example(blocks = blocks)
let cid = (await client.upload(data)).get
let purchaseId = await client.requestStorage(
cid,
expiry = 10.periods,
duration = 20.periods,
nodes = ecNodes,
tolerance = ecTolerance,
collateralPerByte = 1.u256,
pricePerBytePerSecond = minPricePerBytePerSecond,
proofProbability = 1.u256,
)
requestId = (await client.requestId(purchaseId)).get
return purchaseId
proc freeSlot(provider: CodexClient): Future[void] {.async.} =
# Get the second provider signer.
let signer = ethProvider.getSigner(accounts[2])
let marketplaceWithSecondProviderSigner = marketplace.connect(signer)
# Call freeSlot to speed up the process.
# It accelerates the test by skipping validator
# proof verification and not waiting for the full period.
# The downside is that this doesn't reflect the real slot freed process.
let slots = (await provider.getSlots()).get()
let slotId = slotId(requestId, slots[0].slotIndex)
discard await marketplaceWithSecondProviderSigner.freeSlot(slotId)
setup:
filledSlotIds = @[]
freedSlotId = none SlotId
test "repair from local store", test "repair from local store",
NodeConfigs( NodeConfigs(
@ -40,10 +102,7 @@ marketplacesuite "SP Slot Repair":
let provider0 = providers()[0] let provider0 = providers()[0]
let provider1 = providers()[1] let provider1 = providers()[1]
let expiry = 10.periods let expiry = 10.periods
let duration = expiry + 10.periods let duration = 20.periods
let data = await RandomChunker.example(blocks = blocks)
let slotSize = slotSize(blocks, ecNodes, ecTolerance)
# Let's create 2 availabilities # Let's create 2 availabilities
# The first host will be able to host 2 slots with a # The first host will be able to host 2 slots with a
@ -52,62 +111,24 @@ marketplacesuite "SP Slot Repair":
# to host one slot. # to host one slot.
let availability0 = ( let availability0 = (
await provider0.client.postAvailability( await provider0.client.postAvailability(
totalSize = 2 * slotSize.truncate(uint64), totalSize = 2 * size.truncate(uint64),
duration = duration, duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond, minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = 3 * slotSize * collateralPerByte, totalCollateral = 3 * size * collateralPerByte,
) )
).get ).get
let availability1 = ( let availability1 = (
await provider1.client.postAvailability( await provider1.client.postAvailability(
totalSize = slotSize.truncate(uint64), totalSize = size.truncate(uint64),
duration = duration, duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond, minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = slotSize * collateralPerByte, totalCollateral = size * collateralPerByte,
) )
).get ).get
let cid = (await client0.client.upload(data)).get let purchaseId = await createPurchase(client0.client)
let purchaseId = await client0.client.requestStorage(
cid,
expiry = expiry,
duration = duration,
nodes = ecNodes,
tolerance = ecTolerance,
collateralPerByte = 1.u256,
pricePerBytePerSecond = minPricePerBytePerSecond,
proofProbability = 1.u256,
)
let requestId = (await client0.client.requestId(purchaseId)).get
# Here we are keeping track of the slot filled using their ids.
var filledSlotIds: seq[SlotId] = @[]
proc onSlotFilled(eventResult: ?!SlotFilled) =
assert not eventResult.isErr
let event = !eventResult
if event.requestId == requestId:
let slotId = slotId(event.requestId, event.slotIndex)
filledSlotIds.add slotId
let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled)
# Here we are retrieving the slot id freed.
# When the event is triggered, the slot id is removed
# from the filled slot id list.
var freedSlotId = none SlotId
proc onSlotFreed(eventResult: ?!SlotFreed) =
assert not eventResult.isErr
let event = !eventResult
if event.requestId == requestId:
let slotId = slotId(event.requestId, event.slotIndex)
assert slotId in filledSlotIds
filledSlotIds.del(filledSlotIds.find(slotId))
freedSlotId = some(slotId)
let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed) let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
# Wait for purchase starts, meaning that the slots are filled. # Wait for purchase starts, meaning that the slots are filled.
@ -129,10 +150,10 @@ marketplacesuite "SP Slot Repair":
# in order the store the freed slot. # in order the store the freed slot.
await provider0.client.patchAvailability( await provider0.client.patchAvailability(
availabilityId = availability0.id, availabilityId = availability0.id,
totalSize = (3 * slotSize.truncate(uint64)).uint64.some, totalSize = (3 * size.truncate(uint64)).uint64.some,
) )
check eventually(freedSlotId.isSome, timeout = (duration - expiry).int * 1000) await freeSlot(provider1.client)
# We expect that the freed slot is added in the filled slot id list, # We expect that the freed slot is added in the filled slot id list,
# meaning that the slot was repaired by the first host and filled. # meaning that the slot was repaired by the first host and filled.
@ -149,15 +170,10 @@ marketplacesuite "SP Slot Repair":
.init(nodes = 1) .init(nodes = 1)
# .debug() # .debug()
.withLogTopics("node", "erasure").some, .withLogTopics("node", "erasure").some,
providers: CodexConfigs providers: CodexConfigs.init(nodes = 3)
.init(nodes = 3)
.withSimulateProofFailures(idx = 1, failEveryNProofs = 1)
.debug()
.withLogFile()
.withLogTopics("marketplace", "sales", "statemachine", "reservations").some,
validators: CodexConfigs.init(nodes = 1)
# .withLogTopics("validator")
# .debug() # .debug()
# .withLogFile()
# .withLogTopics("marketplace", "sales", "statemachine", "reservations")
.some, .some,
): ):
let client0 = clients()[0] let client0 = clients()[0]
@ -165,79 +181,39 @@ marketplacesuite "SP Slot Repair":
let provider1 = providers()[1] let provider1 = providers()[1]
let provider2 = providers()[2] let provider2 = providers()[2]
let expiry = 10.periods let expiry = 10.periods
let duration = expiry + 10.periods let duration = 20.periods
let data = await RandomChunker.example(blocks = blocks)
let slotSize = slotSize(blocks, ecNodes, ecTolerance)
let datasetSize =
datasetSize(blocks = blocks, nodes = ecNodes, tolerance = ecTolerance)
# Let's create an availability capable to store one slot # Let's create an availability capable to store one slot
# for each SP
let availability0 = ( let availability0 = (
await provider0.client.postAvailability( await provider0.client.postAvailability(
totalSize = slotSize.truncate(uint64), totalSize = size.truncate(uint64),
duration = duration, duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond, minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = 100 * slotSize * collateralPerByte, totalCollateral = 100 * size * collateralPerByte,
) )
).get ).get
let availability1 = ( let availability1 = (
await provider1.client.postAvailability( await provider1.client.postAvailability(
totalSize = slotSize.truncate(uint64), totalSize = size.truncate(uint64),
duration = duration, duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond, minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = 100 * slotSize * collateralPerByte, totalCollateral = 100 * size * collateralPerByte,
) )
).get ).get
discard await provider2.client.postAvailability( discard await provider2.client.postAvailability(
totalSize = slotSize.truncate(uint64), totalSize = size.truncate(uint64),
duration = duration, duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond, minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = 100 * slotSize * collateralPerByte, totalCollateral = 100 * size * collateralPerByte,
) )
let cid = (await client0.client.upload(data)).get let purchaseId = await createPurchase(client0.client)
let purchaseId = await client0.client.requestStorage(
cid,
expiry = expiry,
duration = duration,
nodes = ecNodes,
tolerance = ecTolerance,
collateralPerByte = 1.u256,
pricePerBytePerSecond = minPricePerBytePerSecond,
proofProbability = 1.u256,
)
let requestId = (await client0.client.requestId(purchaseId)).get
# Here we are keeping track of the slot filled using their ids.
var filledSlotIds: seq[SlotId] = @[]
proc onSlotFilled(eventResult: ?!SlotFilled) =
assert not eventResult.isErr
let event = !eventResult
if event.requestId == requestId:
let slotId = slotId(event.requestId, event.slotIndex)
filledSlotIds.add slotId
let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled)
# Here we are retrieving the slot id freed.
# When the event is triggered, the slot id is removed
# from the filled slot id list.
var freedSlotId = none SlotId
proc onSlotFreed(eventResult: ?!SlotFreed) =
assert not eventResult.isErr
let event = !eventResult
let slotId = slotId(event.requestId, event.slotIndex)
if event.requestId == requestId:
assert slotId in filledSlotIds
filledSlotIds.del(filledSlotIds.find(slotId))
freedSlotId = some(slotId)
let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed) let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
# Wait for purchase starts, meaning that the slots are filled.
check eventually( check eventually(
await client0.client.purchaseStateIs(purchaseId, "started"), await client0.client.purchaseStateIs(purchaseId, "started"),
timeout = expiry.int * 1000, timeout = expiry.int * 1000,
@ -246,12 +222,17 @@ marketplacesuite "SP Slot Repair":
# stop client so it doesn't serve any blocks anymore # stop client so it doesn't serve any blocks anymore
await client0.stop() await client0.stop()
# Let's disable the availability of the provider 1 to make sure that he
# will not pick the slot again.
await provider1.client.patchAvailability(availability1.id, enabled = false.some) await provider1.client.patchAvailability(availability1.id, enabled = false.some)
# Let's allow the first SP to host 2 slots.
await provider0.client.patchAvailability( await provider0.client.patchAvailability(
availability0.id, totalSize = (2 * slotSize.truncate(uint64)).some availability0.id, totalSize = (2 * size.truncate(uint64)).some
) )
await freeSlot(provider1.client)
check eventually(freedSlotId.isSome, timeout = expiry.int * 1000) check eventually(freedSlotId.isSome, timeout = expiry.int * 1000)
check eventually(freedSlotId.get in filledSlotIds, timeout = expiry.int * 1000) check eventually(freedSlotId.get in filledSlotIds, timeout = expiry.int * 1000)
@ -262,18 +243,14 @@ marketplacesuite "SP Slot Repair":
test "repair to empty sp", test "repair to empty sp",
NodeConfigs( NodeConfigs(
clients: CodexConfigs.init(nodes = 1) clients: CodexConfigs.init(nodes = 1)
#.debug()
#.withLogTopics("node", "erasure")
.some,
providers: CodexConfigs
.init(nodes = 3)
.withSimulateProofFailures(idx = 1, failEveryNProofs = 1)
.debug()
.withLogFile()
.withLogTopics("marketplace", "sales", "statemachine", "reservations").some,
validators: CodexConfigs.init(nodes = 1)
# .withLogTopics("validator")
# .debug() # .debug()
# .withLogFile()
# .withLogTopics("node", "erasure")
.some,
providers: CodexConfigs.init(nodes = 3)
# .debug()
# .withLogFile()
# .withLogTopics("marketplace", "sales", "statemachine", "reservations")
.some, .some,
): ):
let client0 = clients()[0] let client0 = clients()[0]
@ -283,80 +260,55 @@ marketplacesuite "SP Slot Repair":
let expiry = 10.periods let expiry = 10.periods
let duration = expiry + 10.periods let duration = expiry + 10.periods
let data = await RandomChunker.example(blocks = blocks) # Let's create an availability for only two hosts,
let slotSize = slotSize(blocks, ecNodes, ecTolerance) # the first one will host 2 slots and
# the second one will host 1
discard await provider0.client.postAvailability( discard await provider0.client.postAvailability(
totalSize = 2 * slotSize.truncate(uint64), totalSize = 2 * size.truncate(uint64),
duration = duration, duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond, minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = 100 * slotSize * collateralPerByte, totalCollateral = 100 * size * collateralPerByte,
) )
let availability1 = ( let availability1 = (
await provider1.client.postAvailability( await provider1.client.postAvailability(
totalSize = slotSize.truncate(uint64), totalSize = size.truncate(uint64),
duration = duration, duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond, minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = 100 * slotSize * collateralPerByte, totalCollateral = 100 * size * collateralPerByte,
) )
).get ).get
let cid = (await client0.client.upload(data)).get let purchaseId = await createPurchase(client0.client)
let purchaseId = await client0.client.requestStorage(
cid,
expiry = expiry,
duration = duration,
nodes = ecNodes,
tolerance = ecTolerance,
collateralPerByte = 1.u256,
pricePerBytePerSecond = minPricePerBytePerSecond,
proofProbability = 1.u256,
)
let requestId = (await client0.client.requestId(purchaseId)).get
var filledSlotIds: seq[SlotId] = @[]
proc onSlotFilled(eventResult: ?!SlotFilled) =
assert not eventResult.isErr
let event = !eventResult
if event.requestId == requestId:
let slotId = slotId(event.requestId, event.slotIndex)
filledSlotIds.add slotId
let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled) let filledSubscription = await marketplace.subscribe(SlotFilled, onSlotFilled)
var freedSlotId = none SlotId
proc onSlotFreed(eventResult: ?!SlotFreed) =
assert not eventResult.isErr
let event = !eventResult
let slotId = slotId(event.requestId, event.slotIndex)
if event.requestId == requestId:
assert slotId in filledSlotIds
filledSlotIds.del(filledSlotIds.find(slotId))
freedSlotId = some(slotId)
let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed) let slotFreedsubscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
# Wait for purchase starts, meaning that the slots are filled.
check eventually( check eventually(
await client0.client.purchaseStateIs(purchaseId, "started"), await client0.client.purchaseStateIs(purchaseId, "started"),
timeout = expiry.int * 1000, timeout = expiry.int * 1000,
) )
# stop client so it doesn't serve any blocks anymore
await client0.stop() await client0.stop()
# Let's allow the third SP to participate.
discard await provider2.client.postAvailability( discard await provider2.client.postAvailability(
totalSize = slotSize.truncate(uint64), totalSize = size.truncate(uint64),
duration = duration, duration = duration,
minPricePerBytePerSecond = minPricePerBytePerSecond, minPricePerBytePerSecond = minPricePerBytePerSecond,
totalCollateral = 100 * slotSize * collateralPerByte, totalCollateral = 100 * size * collateralPerByte,
) )
# And let's disable the availability for the second host so he
# will not pick the slot.
await provider1.client.patchAvailability(availability1.id, enabled = false.some) await provider1.client.patchAvailability(availability1.id, enabled = false.some)
check eventually(freedSlotId.isSome, timeout = expiry.int * 1000) # Let's free the slot to speed up the process
await freeSlot(provider1.client)
# At this point, the third SP should repair the repair from the other SP
check eventually(freedSlotId.isSome, timeout = expiry.int * 1000)
check eventually(freedSlotId.get in filledSlotIds, timeout = expiry.int * 1000) check eventually(freedSlotId.get in filledSlotIds, timeout = expiry.int * 1000)
await filledSubscription.unsubscribe() await filledSubscription.unsubscribe()