From bfb82ad2c182b880975cfefaf0785cca5b04de99 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Wed, 16 Oct 2024 22:35:57 +0200 Subject: [PATCH] fine-tune the tests --- .github/workflows/ci.yml | 35 +++-- tests/integration/nodeprocess.nim | 3 +- tests/integration/testmarketplace.nim | 2 - tests/integration/testvalidator.nim | 29 +++-- tests/integration/testvalidator2.nim | 181 ++++++++++++++++++++++++++ 5 files changed, 215 insertions(+), 35 deletions(-) create mode 100644 tests/integration/testvalidator2.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbe18e64..0d27077a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,6 @@ env: cache_nonce: 0 # Allows for easily busting actions/cache caches nim_version: pinned - concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} cancel-in-progress: true @@ -23,23 +22,23 @@ jobs: matrix: ${{ steps.matrix.outputs.matrix }} cache_nonce: ${{ env.cache_nonce }} steps: - - name: Compute matrix - id: matrix - uses: fabiocaccamo/create-matrix-action@v4 - with: - matrix: | - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {macos}, cpu {amd64}, builder {macos-13}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} - os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2} + - name: Compute matrix + id: matrix + uses: fabiocaccamo/create-matrix-action@v4 + with: + matrix: | + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {linux}, cpu {amd64}, builder {ubuntu-20.04}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {macos}, cpu {amd64}, builder {macos-13}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {bash --noprofile --norc -e -o pipefail} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {unittest}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {contract}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {integration}, nim_version {${{ env.nim_version }}}, shell {msys2} + os {windows}, cpu {amd64}, builder {windows-latest}, tests {tools}, nim_version {${{ env.nim_version }}}, shell {msys2} build: needs: matrix diff --git a/tests/integration/nodeprocess.nim b/tests/integration/nodeprocess.nim index 61947c20..4f5cd126 100644 --- a/tests/integration/nodeprocess.nim +++ b/tests/integration/nodeprocess.nim @@ -156,7 +156,8 @@ proc waitUntilStarted*(node: NodeProcess) {.async.} = let started = newFuture[void]() try: discard node.captureOutput(node.startedOutput, started).track(node) - await started.wait(35.seconds) # allow enough time for proof generation + await started.wait(60.seconds) # allow enough time for proof generation + trace "node started" except AsyncTimeoutError: # attempt graceful shutdown in case node was partially started, prevent # zombies diff --git a/tests/integration/testmarketplace.nim b/tests/integration/testmarketplace.nim index 6e62c58b..b37dc933 100644 --- a/tests/integration/testmarketplace.nim +++ b/tests/integration/testmarketplace.nim @@ -1,5 +1,3 @@ -import pkg/stew/byteutils -import pkg/codex/units import ../examples import ../contracts/time import ../contracts/deployment diff --git a/tests/integration/testvalidator.nim b/tests/integration/testvalidator.nim index 04be5bf5..f87766d6 100644 --- a/tests/integration/testvalidator.nim +++ b/tests/integration/testvalidator.nim @@ -25,18 +25,18 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) .withEthProvider("http://localhost:8545") - .debug() # uncomment to enable console log output - # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - # .withLogTopics("node", "marketplace", "clock") - .withLogTopics("node", "purchases", "slotqueue", "market") + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("purchases", "onchain") .some, providers: CodexConfigs.init(nodes=1) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .withEthProvider("http://localhost:8545") + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("sales", "onchain") .some, validators: @@ -44,7 +44,7 @@ marketplacesuite "Validation": .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log .withLogTopics("validator") # each topic as a separate string argument .some @@ -102,17 +102,18 @@ marketplacesuite "Validation": clients: CodexConfigs.init(nodes=1) .withEthProvider("http://localhost:8545") - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("node", "purchases", "slotqueue", "market") + .withLogTopics("purchases", "onchain") .some, providers: CodexConfigs.init(nodes=1) .withSimulateProofFailures(idx=0, failEveryNProofs=1) - .debug() # uncomment to enable console log output - .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log - .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .withEthProvider("http://localhost:8545") + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("sales", "onchain") .some ): let client0 = clients()[0].client @@ -153,7 +154,7 @@ marketplacesuite "Validation": .withValidationGroups(groups = 2) .withValidationGroupIndex(idx = 0, groupIndex = 0) .withValidationGroupIndex(idx = 1, groupIndex = 1) - .debug() # uncomment to enable console log output + # .debug() # uncomment to enable console log output .withLogFile() # uncomment to output log file to: # tests/integration/logs/ //_.log .withLogTopics("validator") # each topic as a separate string argument diff --git a/tests/integration/testvalidator2.nim b/tests/integration/testvalidator2.nim new file mode 100644 index 00000000..b0e92188 --- /dev/null +++ b/tests/integration/testvalidator2.nim @@ -0,0 +1,181 @@ +from std/times import inMilliseconds, initDuration, inSeconds, fromUnix +import pkg/codex/logutils +import ../contracts/time +import ../contracts/deployment +import ../codex/helpers +import ../examples +import ./marketplacesuite +import ./nodeconfigs + +export logutils + +logScope: + topics = "integration test validation" + +marketplacesuite "Validation": + let nodes = 3 + let tolerance = 1 + let proofProbability = 1 + + test "validator marks proofs as missing when using validation groups", NodeConfigs( + # Uncomment to start Hardhat automatically, typically so logs can be inspected locally + hardhat: + HardhatConfig.none, + + clients: + CodexConfigs.init(nodes=1) + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "clock") + .withLogTopics("node", "purchases", "slotqueue", "market") + .some, + + providers: + CodexConfigs.init(nodes=1) + .withSimulateProofFailures(idx=0, failEveryNProofs=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .some, + + validators: + CodexConfigs.init(nodes=2) + .withValidationGroups(groups = 2) + .withValidationGroupIndex(idx = 0, groupIndex = 0) + .withValidationGroupIndex(idx = 1, groupIndex = 1) + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + .withLogTopics("validator") # each topic as a separate string argument + .some + ): + let client0 = clients()[0].client + let expiry = 5.periods + let duration = expiry + 10.periods + + var currentTime = await ethProvider.currentTime() + let requestEndTime = currentTime.truncate(uint64) + duration + + let data = await RandomChunker.example(blocks=8) + + # TODO: better value for data.len below. This TODO is also present in + # testproofs.nim - we may want to address it or remove the comment. + createAvailabilities(data.len * 2, duration) + + let cid = client0.upload(data).get + + let purchaseId = await client0.requestStorage( + cid, + expiry=expiry, + duration=duration, + nodes=nodes, + tolerance=tolerance, + proofProbability=proofProbability + ) + let requestId = client0.requestId(purchaseId).get + + debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId + + check eventually(client0.purchaseStateIs(purchaseId, "started"), + timeout = expiry.int * 1000) + + currentTime = await ethProvider.currentTime() + let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int + + debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds + + # Because of Erasure Coding, the expected number of slots being freed + # is tolerance + 1. When more than tolerance slots are freed, the whole + # request will fail. Thus, awaiting for a failing state should + # be sufficient to conclude that validators did their job correctly. + # NOTICE: We actually have to wait for the "errored" state, because + # immediately after withdrawing the funds the purchasing state machine + # transitions to the "errored" state. + check eventually(client0.purchaseStateIs(purchaseId, "errored"), + timeout = (secondsTillRequestEnd + 60) * 1000) + + test "validator uses historical state to mark missing proofs", NodeConfigs( + # Uncomment to start Hardhat automatically, typically so logs can be inspected locally + hardhat: + HardhatConfig.none, + + clients: + CodexConfigs.init(nodes=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("node", "marketplace", "clock") + # .withLogTopics("node", "purchases", "slotqueue", "market") + .some, + + providers: + CodexConfigs.init(nodes=1) + .withSimulateProofFailures(idx=0, failEveryNProofs=1) + # .debug() # uncomment to enable console log output + # .withLogFile() # uncomment to output log file to tests/integration/logs/ //_.log + # .withLogTopics("marketplace", "sales", "reservations", "node", "clock", "slotsbuilder") + .some + ): + let client0 = clients()[0].client + let expiry = 5.periods + let duration = expiry + 10.periods + + var currentTime = await ethProvider.currentTime() + let requestEndTime = currentTime.truncate(uint64) + duration + + let data = await RandomChunker.example(blocks=8) + + # TODO: better value for data.len below. This TODO is also present in + # testproofs.nim - we may want to address it or remove the comment. + createAvailabilities(data.len * 2, duration) + + let cid = client0.upload(data).get + + let purchaseId = await client0.requestStorage( + cid, + expiry=expiry, + duration=duration, + nodes=nodes, + tolerance=tolerance, + proofProbability=proofProbability + ) + let requestId = client0.requestId(purchaseId).get + + debug "validation suite", purchaseId = purchaseId.toHex, requestId = requestId + + check eventually(client0.purchaseStateIs(purchaseId, "started"), + timeout = expiry.int * 1000) + + # just to make sure we have a mined block that separates us + # from the block containing the last SlotFilled event + discard await ethProvider.send("evm_mine") + + var validators = CodexConfigs.init(nodes=2) + .withValidationGroups(groups = 2) + .withValidationGroupIndex(idx = 0, groupIndex = 0) + .withValidationGroupIndex(idx = 1, groupIndex = 1) + # .debug() # uncomment to enable console log output + .withLogFile() # uncomment to output log file to: + # tests/integration/logs/ //_.log + .withLogTopics("validator") # each topic as a separate string argument + + failAndTeardownOnError "failed to start validator nodes": + for config in validators.configs.mitems: + let node = await startValidatorNode(config) + running.add RunningNode( + role: Role.Validator, + node: node + ) + + currentTime = await ethProvider.currentTime() + let secondsTillRequestEnd = (requestEndTime - currentTime.truncate(uint64)).int + + debug "validation suite", secondsTillRequestEnd = secondsTillRequestEnd.seconds + + # Because of Erasure Coding, the expected number of slots being freed + # is tolerance + 1. When more than tolerance slots are freed, the whole + # request will fail. Thus, awaiting for a failing state should + # be sufficient to conclude that validators did their job correctly. + # NOTICE: We actually have to wait for the "errored" state, because + # immediately after withdrawing the funds the purchasing state machine + # transitions to the "errored" state. + check eventually(client0.purchaseStateIs(purchaseId, "errored"), + timeout = (secondsTillRequestEnd + 60) * 1000)