review and refactor validator integration tests
This commit is contained in:
Marcin Czenko
parent
2404b75f0d
commit
60142ea061
|
|
@ -1,4 +1,5 @@
|
||||||
from std/times import inMilliseconds, initDuration, inSeconds, fromUnix
|
from std/times import inMilliseconds, initDuration, inSeconds, fromUnix
|
||||||
|
import std/sets
|
||||||
import pkg/codex/logutils
|
import pkg/codex/logutils
|
||||||
import ../contracts/time
|
import ../contracts/time
|
||||||
import ../contracts/deployment
|
import ../contracts/deployment
|
||||||
|
|
@ -13,188 +14,50 @@ logScope:
|
||||||
topics = "integration test validation"
|
topics = "integration test validation"
|
||||||
|
|
||||||
marketplacesuite "Validaton":
|
marketplacesuite "Validaton":
|
||||||
|
let nodes = 3
|
||||||
|
let tolerance = 1
|
||||||
|
var slotsFilled: seq[SlotId]
|
||||||
|
var slotsFreed: seq[SlotId]
|
||||||
|
|
||||||
test "validator uses historical state to mark missing proofs", NodeConfigs(
|
proc trackSlotsFilled(marketplace: Marketplace):
|
||||||
# Uncomment to start Hardhat automatically, typically so logs can be inspected locally
|
Future[provider.Subscription] {.async.} =
|
||||||
hardhat:
|
slotsFilled = newSeq[SlotId]()
|
||||||
HardhatConfig.none,
|
|
||||||
|
|
||||||
clients:
|
|
||||||
CodexConfigs.init(nodes=1)
|
|
||||||
# .debug() # uncomment to enable console log output
|
|
||||||
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
|
||||||
# .withLogTopics("node", "marketplace", "clock")
|
|
||||||
.some,
|
|
||||||
|
|
||||||
providers:
|
|
||||||
CodexConfigs.init(nodes=1)
|
|
||||||
.withSimulateProofFailures(idx=0, failEveryNProofs=1)
|
|
||||||
# .debug() # uncomment to enable console log output
|
|
||||||
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
|
||||||
# .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder")
|
|
||||||
.some
|
|
||||||
):
|
|
||||||
let client0 = clients()[0].client
|
|
||||||
let expiry = 5.periods
|
|
||||||
let duration = expiry + 10.periods
|
|
||||||
|
|
||||||
let data = await RandomChunker.example(blocks=8)
|
|
||||||
createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
|
|
||||||
|
|
||||||
let cid = client0.upload(data).get
|
|
||||||
|
|
||||||
let purchaseId = await client0.requestStorage(
|
|
||||||
cid,
|
|
||||||
expiry=expiry,
|
|
||||||
duration=duration,
|
|
||||||
nodes=3,
|
|
||||||
tolerance=1,
|
|
||||||
proofProbability=1
|
|
||||||
)
|
|
||||||
let requestId = client0.requestId(purchaseId).get
|
|
||||||
|
|
||||||
check eventually(client0.purchaseStateIs(purchaseId, "started"),
|
|
||||||
timeout = expiry.int * 1000)
|
|
||||||
|
|
||||||
var validators = CodexConfigs.init(nodes=2)
|
|
||||||
.withValidationGroups(groups = 2)
|
|
||||||
.withValidationGroupIndex(idx = 0, groupIndex = 0)
|
|
||||||
.withValidationGroupIndex(idx = 1, groupIndex = 1)
|
|
||||||
.debug() # uncomment to enable console log output
|
|
||||||
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
|
||||||
.withLogTopics("validator") # each topic as a separate string argument
|
|
||||||
|
|
||||||
failAndTeardownOnError "failed to start validator nodes":
|
|
||||||
for config in validators.configs.mitems:
|
|
||||||
let node = await startValidatorNode(config)
|
|
||||||
running.add RunningNode(
|
|
||||||
role: Role.Validator,
|
|
||||||
node: node
|
|
||||||
)
|
|
||||||
|
|
||||||
var slotWasFreed = false
|
|
||||||
proc onSlotFreed(event: SlotFreed) =
|
|
||||||
if event.requestId == requestId:
|
|
||||||
slotWasFreed = true
|
|
||||||
|
|
||||||
let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
|
|
||||||
|
|
||||||
check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000)
|
|
||||||
|
|
||||||
await subscription.unsubscribe()
|
|
||||||
|
|
||||||
test "validator only looks 30 days back for historical state", NodeConfigs(
|
|
||||||
# Uncomment to start Hardhat automatically, typically so logs can be inspected locally
|
|
||||||
hardhat:
|
|
||||||
HardhatConfig.none,
|
|
||||||
|
|
||||||
clients:
|
|
||||||
CodexConfigs.init(nodes=1)
|
|
||||||
# .debug() # uncomment to enable console log output
|
|
||||||
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
|
||||||
# .withLogTopics("node", "marketplace", "clock")
|
|
||||||
.some,
|
|
||||||
|
|
||||||
providers:
|
|
||||||
CodexConfigs.init(nodes=1)
|
|
||||||
.withSimulateProofFailures(idx=0, failEveryNProofs=1)
|
|
||||||
# .debug() # uncomment to enable console log output
|
|
||||||
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
|
||||||
# .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder")
|
|
||||||
.some
|
|
||||||
):
|
|
||||||
let client0 = clients()[0].client
|
|
||||||
let expiry = 5.periods
|
|
||||||
let duration30days = initDuration(days = 30)
|
|
||||||
let duration = expiry + duration30days.inSeconds.uint64 + 10.periods
|
|
||||||
|
|
||||||
let data = await RandomChunker.example(blocks=8)
|
|
||||||
createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
|
|
||||||
|
|
||||||
let cid = client0.upload(data).get
|
|
||||||
|
|
||||||
var currentTime = await ethProvider.currentTime()
|
|
||||||
|
|
||||||
let expiryEndTime = currentTime.truncate(uint64) + expiry
|
|
||||||
let requestEndTime = currentTime.truncate(uint64) + duration
|
|
||||||
debug "test validator: ", currentTime = currentTime.truncate(uint64),
|
|
||||||
requestEndTime = requestEndTime, expiryEndTime = expiryEndTime
|
|
||||||
debug "test validator: ", currentTime = currentTime.truncate(int64).fromUnix,
|
|
||||||
requestEndTime = requestEndTime.int64.fromUnix,
|
|
||||||
expiryEndTime = expiryEndTime.int64.fromUnix
|
|
||||||
|
|
||||||
proc onSlotFilled(event: SlotFilled) =
|
proc onSlotFilled(event: SlotFilled) =
|
||||||
let slotId = slotId(event.requestId, event.slotIndex)
|
let slotId = slotId(event.requestId, event.slotIndex)
|
||||||
|
slotsFilled.add(slotId)
|
||||||
debug "SlotFilled", requestId = event.requestId, slotIndex = event.slotIndex,
|
debug "SlotFilled", requestId = event.requestId, slotIndex = event.slotIndex,
|
||||||
slotId = slotId
|
slotId = slotId
|
||||||
|
|
||||||
let subscriptionOnSlotFilled = await marketplace.subscribe(SlotFilled, onSlotFilled)
|
let subscription = await marketplace.subscribe(SlotFilled, onSlotFilled)
|
||||||
|
subscription
|
||||||
let purchaseId = await client0.requestStorage(
|
|
||||||
cid,
|
proc trackSlotsFreed(requestId: RequestId, marketplace: Marketplace):
|
||||||
expiry=expiry,
|
Future[provider.Subscription] {.async.} =
|
||||||
duration=duration,
|
slotsFreed = newSeq[SlotId]()
|
||||||
nodes=3,
|
|
||||||
tolerance=1,
|
|
||||||
reward=1.u256,
|
|
||||||
proofProbability=1
|
|
||||||
)
|
|
||||||
let requestId = client0.requestId(purchaseId).get
|
|
||||||
|
|
||||||
check eventually(client0.purchaseStateIs(purchaseId, "started"),
|
|
||||||
timeout = expiry.int * 1000)
|
|
||||||
|
|
||||||
currentTime = await ethProvider.currentTime()
|
|
||||||
var waitTime = (expiryEndTime - currentTime.truncate(uint64)).int.seconds
|
|
||||||
debug "test validation - waiting till end of expiry", waitTime = waitTime
|
|
||||||
await sleepAsync(waitTime)
|
|
||||||
|
|
||||||
discard await ethProvider.send("evm_mine")
|
|
||||||
|
|
||||||
await ethProvider.advanceTimeTo(
|
|
||||||
expiryEndTime.u256 + duration30days.inSeconds.u256)
|
|
||||||
debug "test validator[after advance]: ", currentTime = currentTime.truncate(SecondsSince1970)
|
|
||||||
debug "test validator[after advance]: ", currentTime =
|
|
||||||
currentTime.truncate(SecondsSince1970).fromUnix
|
|
||||||
|
|
||||||
var validators = CodexConfigs.init(nodes=1)
|
|
||||||
.debug() # uncomment to enable console log output
|
|
||||||
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
|
||||||
.withLogTopics("validator", "clock", "market") # each topic as a separate string argument
|
|
||||||
|
|
||||||
failAndTeardownOnError "failed to start validator nodes":
|
|
||||||
for config in validators.configs.mitems:
|
|
||||||
let node = await startValidatorNode(config)
|
|
||||||
running.add RunningNode(
|
|
||||||
role: Role.Validator,
|
|
||||||
node: node
|
|
||||||
)
|
|
||||||
|
|
||||||
var slotWasFreed = false
|
|
||||||
proc onSlotFreed(event: SlotFreed) =
|
proc onSlotFreed(event: SlotFreed) =
|
||||||
if event.requestId == requestId:
|
if event.requestId == requestId:
|
||||||
slotWasFreed = true
|
let slotId = slotId(event.requestId, event.slotIndex)
|
||||||
|
slotsFreed.add(slotId)
|
||||||
|
debug "onSlotFreed", requestId = requestId, slotIndex = event.slotIndex,
|
||||||
|
slotId = slotId, slotsFreed = slotsFreed.len
|
||||||
|
|
||||||
let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
|
let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
|
||||||
|
subscription
|
||||||
|
|
||||||
# check not freed
|
proc checkSlotsFailed(slotsFilled: seq[SlotId],
|
||||||
currentTime = await ethProvider.currentTime()
|
slotsFreed: seq[SlotId], marketplace: Marketplace) {.async.} =
|
||||||
if requestEndTime > currentTime.truncate(uint64):
|
let slotsNotFreed = slotsFilled.filter(
|
||||||
waitTime = (requestEndTime - currentTime.truncate(uint64)).int.seconds
|
slotId => not slotsFreed.contains(slotId)
|
||||||
debug "test validation - waiting for request end", waitTime = waitTime
|
).toHashSet
|
||||||
await sleepAsync(waitTime)
|
var slotsFailed = initHashSet[SlotId]()
|
||||||
|
for slotId in slotsFilled:
|
||||||
|
let state = await marketplace.slotState(slotId)
|
||||||
|
if state == SlotState.Failed:
|
||||||
|
slotsFailed.incl(slotId)
|
||||||
|
|
||||||
debug "test validation - request ended"
|
debug "slots failed", slotsFailed = slotsFailed, slotsNotFreed = slotsNotFreed
|
||||||
|
check slotsNotFreed == slotsFailed
|
||||||
check not slotWasFreed
|
|
||||||
|
|
||||||
# check eventually(client0.purchaseStateIs(purchaseId, "finished"),
|
|
||||||
# timeout = 60 * 1000)
|
|
||||||
|
|
||||||
await subscription.unsubscribe()
|
|
||||||
await subscriptionOnSlotFilled.unsubscribe()
|
|
||||||
debug "test validation - unsubscribed"
|
|
||||||
|
|
||||||
test "validator marks proofs as missing when using validation groups", NodeConfigs(
|
test "validator marks proofs as missing when using validation groups", NodeConfigs(
|
||||||
# Uncomment to start Hardhat automatically, typically so logs can be inspected locally
|
# Uncomment to start Hardhat automatically, typically so logs can be inspected locally
|
||||||
hardhat:
|
hardhat:
|
||||||
|
|
@ -220,9 +83,9 @@ marketplacesuite "Validaton":
|
||||||
.withValidationGroups(groups = 2)
|
.withValidationGroups(groups = 2)
|
||||||
.withValidationGroupIndex(idx = 0, groupIndex = 0)
|
.withValidationGroupIndex(idx = 0, groupIndex = 0)
|
||||||
.withValidationGroupIndex(idx = 1, groupIndex = 1)
|
.withValidationGroupIndex(idx = 1, groupIndex = 1)
|
||||||
.debug() # uncomment to enable console log output
|
# .debug() # uncomment to enable console log output
|
||||||
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
||||||
.withLogTopics("validator")
|
# .withLogTopics("validator")
|
||||||
# .withLogTopics("validator", "integration", "ethers", "clock")
|
# .withLogTopics("validator", "integration", "ethers", "clock")
|
||||||
.some
|
.some
|
||||||
):
|
):
|
||||||
|
|
@ -231,7 +94,12 @@ marketplacesuite "Validaton":
|
||||||
let duration = expiry + 10.periods
|
let duration = expiry + 10.periods
|
||||||
|
|
||||||
let data = await RandomChunker.example(blocks=8)
|
let data = await RandomChunker.example(blocks=8)
|
||||||
createAvailabilities(data.len * 2, duration) # TODO: better value for data.len
|
|
||||||
|
# TODO: better value for data.len below. This TODO is also present in
|
||||||
|
# testproofs.nim - we may want to address it or remove the comment.
|
||||||
|
createAvailabilities(data.len * 2, duration)
|
||||||
|
|
||||||
|
let slotFilledSubscription = await trackSlotsFilled(marketplace)
|
||||||
|
|
||||||
let cid = client0.upload(data).get
|
let cid = client0.upload(data).get
|
||||||
|
|
||||||
|
|
@ -239,21 +107,109 @@ marketplacesuite "Validaton":
|
||||||
cid,
|
cid,
|
||||||
expiry=expiry,
|
expiry=expiry,
|
||||||
duration=duration,
|
duration=duration,
|
||||||
nodes=3,
|
nodes=nodes,
|
||||||
tolerance=1,
|
tolerance=tolerance,
|
||||||
proofProbability=1
|
proofProbability=1
|
||||||
)
|
)
|
||||||
let requestId = client0.requestId(purchaseId).get
|
let requestId = client0.requestId(purchaseId).get
|
||||||
|
|
||||||
check eventually(client0.purchaseStateIs(purchaseId, "started"), timeout = expiry.int * 1000)
|
check eventually(client0.purchaseStateIs(purchaseId, "started"),
|
||||||
|
timeout = expiry.int * 1000)
|
||||||
|
|
||||||
|
let slotFreedSubscription =
|
||||||
|
await trackSlotsFreed(requestId, marketplace)
|
||||||
|
|
||||||
var slotWasFreed = false
|
let expectedSlotsFreed = nodes - tolerance
|
||||||
proc onSlotFreed(event: SlotFreed) =
|
check eventually((slotsFreed.len == expectedSlotsFreed),
|
||||||
if event.requestId == requestId:
|
timeout=(duration - expiry).int * 1000)
|
||||||
slotWasFreed = true
|
|
||||||
|
# Because of erasure coding, if e.g. 2 out of 3 nodes are freed, the last
|
||||||
|
# node will not be freed but marked as "Failed" because the whole request
|
||||||
|
# will fail. For this reason we need an extra check:
|
||||||
|
await checkSlotsFailed(slotsFilled, slotsFreed, marketplace)
|
||||||
|
|
||||||
|
await slotFilledSubscription.unsubscribe()
|
||||||
|
await slotFreedSubscription.unsubscribe()
|
||||||
|
|
||||||
|
test "validator uses historical state to mark missing proofs", NodeConfigs(
|
||||||
|
# Uncomment to start Hardhat automatically, typically so logs can be inspected locally
|
||||||
|
hardhat:
|
||||||
|
HardhatConfig.none,
|
||||||
|
|
||||||
let subscription = await marketplace.subscribe(SlotFreed, onSlotFreed)
|
clients:
|
||||||
|
CodexConfigs.init(nodes=1)
|
||||||
|
# .debug() # uncomment to enable console log output
|
||||||
|
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
||||||
|
# .withLogTopics("node", "marketplace", "clock")
|
||||||
|
.some,
|
||||||
|
|
||||||
check eventually(slotWasFreed, timeout=(duration - expiry).int * 1000)
|
providers:
|
||||||
|
CodexConfigs.init(nodes=1)
|
||||||
|
.withSimulateProofFailures(idx=0, failEveryNProofs=1)
|
||||||
|
# .debug() # uncomment to enable console log output
|
||||||
|
# .withLogFile() # uncomment to output log file to tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
||||||
|
# .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder")
|
||||||
|
.some
|
||||||
|
):
|
||||||
|
let client0 = clients()[0].client
|
||||||
|
let expiry = 5.periods
|
||||||
|
let duration = expiry + 10.periods
|
||||||
|
|
||||||
await subscription.unsubscribe()
|
let data = await RandomChunker.example(blocks=8)
|
||||||
|
|
||||||
|
# TODO: better value for data.len below. This TODO is also present in
|
||||||
|
# testproofs.nim - we may want to address it or remove the comment.
|
||||||
|
createAvailabilities(data.len * 2, duration)
|
||||||
|
|
||||||
|
let slotFilledSubscription = await trackSlotsFilled(marketplace)
|
||||||
|
|
||||||
|
let cid = client0.upload(data).get
|
||||||
|
|
||||||
|
let purchaseId = await client0.requestStorage(
|
||||||
|
cid,
|
||||||
|
expiry=expiry,
|
||||||
|
duration=duration,
|
||||||
|
nodes=nodes,
|
||||||
|
tolerance=tolerance,
|
||||||
|
proofProbability=1
|
||||||
|
)
|
||||||
|
let requestId = client0.requestId(purchaseId).get
|
||||||
|
|
||||||
|
check eventually(client0.purchaseStateIs(purchaseId, "started"),
|
||||||
|
timeout = expiry.int * 1000)
|
||||||
|
|
||||||
|
# just to make sure we have a mined block that separates us
|
||||||
|
# from the block containing the last SlotFilled event
|
||||||
|
discard await ethProvider.send("evm_mine")
|
||||||
|
|
||||||
|
var validators = CodexConfigs.init(nodes=2)
|
||||||
|
.withValidationGroups(groups = 2)
|
||||||
|
.withValidationGroupIndex(idx = 0, groupIndex = 0)
|
||||||
|
.withValidationGroupIndex(idx = 1, groupIndex = 1)
|
||||||
|
# .debug() # uncomment to enable console log output
|
||||||
|
# .withLogFile() # uncomment to output log file to:
|
||||||
|
# tests/integration/logs/<start_datetime> <suite_name>/<test_name>/<node_role>_<node_idx>.log
|
||||||
|
# .withLogTopics("validator") # each topic as a separate string argument
|
||||||
|
|
||||||
|
failAndTeardownOnError "failed to start validator nodes":
|
||||||
|
for config in validators.configs.mitems:
|
||||||
|
let node = await startValidatorNode(config)
|
||||||
|
running.add RunningNode(
|
||||||
|
role: Role.Validator,
|
||||||
|
node: node
|
||||||
|
)
|
||||||
|
|
||||||
|
let slotFreedSubscription =
|
||||||
|
await trackSlotsFreed(requestId, marketplace)
|
||||||
|
|
||||||
|
let expectedSlotsFreed = nodes - tolerance
|
||||||
|
check eventually((slotsFreed.len == expectedSlotsFreed),
|
||||||
|
timeout=(duration - expiry).int * 1000)
|
||||||
|
|
||||||
|
# Because of erasure coding, if e.g. 2 out of 3 nodes are freed, the last
|
||||||
|
# node will not be freed but marked as "Failed" because the whole request
|
||||||
|
# will fail. For this reason we need an extra check:
|
||||||
|
await checkSlotsFailed(slotsFilled, slotsFreed, marketplace)
|
||||||
|
|
||||||
|
await slotFilledSubscription.unsubscribe()
|
||||||
|
await slotFreedSubscription.unsubscribe()
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue