From 83333658489bc7a5d2533424a20d52f7ffd914d7 Mon Sep 17 00:00:00 2001 From: tersec Date: Fri, 2 Aug 2024 05:30:22 +0000 Subject: [PATCH] replace validator Bloom filter with partial bucket sort (#6469) --- AllTests-mainnet.md | 6 +- beacon_chain/bloomfilter.nim | 49 ---- beacon_chain/spec/state_transition_block.nim | 110 ++++---- beacon_chain/validator_bucket_sort.nim | 90 ++++++ ncli/ncli_common.nim | 12 +- nfuzz/libnfuzz.nim | 4 +- tests/all_tests.nim | 4 +- .../altair/test_fixture_operations.nim | 7 +- .../bellatrix/test_fixture_operations.nim | 7 +- .../capella/test_fixture_operations.nim | 7 +- .../deneb/test_fixture_operations.nim | 7 +- .../electra/test_fixture_operations.nim | 19 +- .../phase0/test_fixture_operations.nim | 7 +- tests/test_bloom_filter.nim | 147 ---------- tests/test_validator_bucket_sort.nim | 260 ++++++++++++++++++ tests/teststateutil.nim | 10 +- 16 files changed, 453 insertions(+), 293 deletions(-) delete mode 100644 beacon_chain/bloomfilter.nim create mode 100644 beacon_chain/validator_bucket_sort.nim delete mode 100644 tests/test_bloom_filter.nim create mode 100644 tests/test_validator_bucket_sort.nim diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index f931b3b70..822869e4b 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -933,10 +933,10 @@ OK: 6/6 Fail: 0/6 Skip: 0/6 + Dynamic validator set: updateDynamicValidators() test OK ``` OK: 4/4 Fail: 0/4 Skip: 0/4 -## ValidatorPubKey Bloom filter +## ValidatorPubKey bucket sort ```diff -+ incremental construction with no false positives/negatives OK -+ one-shot construction with no false positives/negatives OK ++ incremental construction OK ++ one-shot construction OK ``` OK: 2/2 Fail: 0/2 Skip: 0/2 ## Zero signature sanity checks diff --git a/beacon_chain/bloomfilter.nim b/beacon_chain/bloomfilter.nim deleted file mode 100644 index a4a966329..000000000 --- a/beacon_chain/bloomfilter.nim +++ /dev/null @@ -1,49 +0,0 @@ -# beacon_chain -# Copyright (c) 2024 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [].} - -import "."/spec/crypto - -from stew/bitops2 import getBit, setBit -from "."/spec/datatypes/base import Validator, pubkey -from "."/spec/helpers import bytes_to_uint32 - -const - # https://hur.st/bloomfilter/?n=4M&p=&m=8MiB&k= - pubkeyBloomFilterScale = 23 # 21 too small, 22 borderline, 24 also ok - -type - PubkeyBloomFilter* = object - data: array[1 shl pubkeyBloomFilterScale, byte] - -iterator bloomFilterHashes(pubkey: ValidatorPubKey): auto = - const pubkeyBloomFilterMask = (1 shl pubkeyBloomFilterScale) - 1 - for r in countup(0'u32, 20'u32, 4'u32): - # ValidatorPubKeys have fairly uniform entropy; using enough hash - # functions also reduces risk of low-entropy portions - yield pubkey.blob.toOpenArray(r, r+3).bytes_to_uint32 and - pubkeyBloomFilterMask - -template incl*(bloomFilter: var PubkeyBloomFilter, pubkey: ValidatorPubKey) = - for bloomFilterHash in bloomFilterHashes(pubkey): - setBit(bloomFilter.data, bloomFilterHash) - -func constructBloomFilter*(x: openArray[Validator]): auto = - let res = new PubkeyBloomFilter - for m in x: - incl(res[], m.pubkey) - res - -func mightContain*( - bloomFilter: PubkeyBloomFilter, pubkey: ValidatorPubKey): bool = - # Might return false positive, but never false negative - for bloomFilterHash in bloomFilterHashes(pubkey): - if not getBit(bloomFilter.data, bloomFilterHash): - return false - - true diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index eb8bd16b0..a4e695ca1 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -275,48 +275,20 @@ proc process_attester_slashing*( ok((proposer_reward, cur_exit_queue_info)) -func findValidatorIndex*(state: ForkyBeaconState, pubkey: ValidatorPubKey): - Opt[ValidatorIndex] = - # This linear scan is unfortunate, but should be fairly fast as we do a simple - # byte comparison of the key. The alternative would be to build a Table, but - # given that each block can hold no more than 16 deposits, it's slower to - # build the table and use it for lookups than to scan it like this. - # Once we have a reusable, long-lived cache, this should be revisited - # - # For deposit processing purposes, two broad cases exist, either - # - # (a) someone has deposited all 32 required ETH as a single transaction, - # in which case the index doesn't yet exist so the search order does - # not matter so long as it's generally in an order memory controller - # prefetching can predict; or - # - # (b) the deposit has been split into multiple parts, typically not far - # apart from each other, such that on average one would expect this - # validator index to be nearer the maximal than minimal index. - # - # countdown() infinite-loops if the lower bound with uint32 is 0, so - # shift indices by 1, which avoids triggering unsigned wraparound. - for vidx in countdown(state.validators.len.uint32, 1): - if state.validators.asSeq[vidx - 1].pubkey == pubkey: - return Opt[ValidatorIndex].ok((vidx - 1).ValidatorIndex) - -from ".."/bloomfilter import - PubkeyBloomFilter, constructBloomFilter, incl, mightContain +from ".."/validator_bucket_sort import + BucketSortedValidators, add, findValidatorIndex, sortValidatorBuckets # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.0/specs/phase0/beacon-chain.md#deposits # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.0/specs/electra/beacon-chain.md#updated--apply_deposit proc apply_deposit( cfg: RuntimeConfig, state: var ForkyBeaconState, - bloom_filter: var PubkeyBloomFilter, deposit_data: DepositData, - flags: UpdateFlags): Result[void, cstring] = + bucketSortedValidators: var BucketSortedValidators, + deposit_data: DepositData, flags: UpdateFlags): Result[void, cstring] = let pubkey = deposit_data.pubkey amount = deposit_data.amount - index = - if bloom_filter.mightContain(pubkey): - findValidatorIndex(state, pubkey) - else: - Opt.none(ValidatorIndex) + index = findValidatorIndex( + state.validators.asSeq, bucketSortedValidators, pubkey) if index.isSome(): # Increase balance by deposit amount @@ -358,14 +330,15 @@ proc apply_deposit( return err("apply_deposit: too many validators (current_epoch_participation)") if not state.inactivity_scores.add(0'u64): return err("apply_deposit: too many validators (inactivity_scores)") + let new_vidx = state.validators.lenu64 - 1 when typeof(state).kind >= ConsensusFork.Electra: debugComment "check hashlist add return" # [New in Electra:EIP7251] discard state.pending_balance_deposits.add PendingBalanceDeposit( - index: state.validators.lenu64 - 1, amount: amount) + index: new_vidx, amount: amount) doAssert state.validators.len == state.balances.len - bloom_filter.incl pubkey + bucketSortedValidators.add new_vidx.ValidatorIndex else: # Deposits may come with invalid signatures - in that case, they are not # turned into a validator but still get processed to keep the deposit @@ -378,7 +351,8 @@ proc apply_deposit( # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.0/specs/phase0/beacon-chain.md#deposits proc process_deposit*( cfg: RuntimeConfig, state: var ForkyBeaconState, - bloom_filter: var PubkeyBloomFilter, deposit: Deposit, flags: UpdateFlags): + bucketSortedValidators: var BucketSortedValidators, + deposit: Deposit, flags: UpdateFlags): Result[void, cstring] = ## Process an Eth1 deposit, registering a validator or increasing its balance. @@ -395,12 +369,13 @@ proc process_deposit*( # Deposits must be processed in order state.eth1_deposit_index += 1 - apply_deposit(cfg, state, bloom_filter, deposit.data, flags) + apply_deposit(cfg, state, bucketSortedValidators, deposit.data, flags) # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.3/specs/electra/beacon-chain.md#new-process_deposit_request func process_deposit_request*( cfg: RuntimeConfig, state: var electra.BeaconState, - bloom_filter: var PubkeyBloomFilter, deposit_request: DepositRequest, + bucketSortedValidators: var BucketSortedValidators, + deposit_request: DepositRequest, flags: UpdateFlags): Result[void, cstring] = # Set deposit request start index if state.deposit_requests_start_index == @@ -408,7 +383,7 @@ func process_deposit_request*( state.deposit_requests_start_index = deposit_request.index apply_deposit( - cfg, state, bloom_filter, DepositData( + cfg, state, bucketSortedValidators, DepositData( pubkey: deposit_request.pubkey, withdrawal_credentials: deposit_request.withdrawal_credentials, amount: deposit_request.amount, @@ -510,6 +485,7 @@ proc process_bls_to_execution_change*( # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.3/specs/electra/beacon-chain.md#new-process_withdrawal_request func process_withdrawal_request*( cfg: RuntimeConfig, state: var electra.BeaconState, + bucketSortedValidators: BucketSortedValidators, withdrawal_request: WithdrawalRequest, cache: var StateCache) = let amount = withdrawal_request.amount @@ -523,7 +499,9 @@ func process_withdrawal_request*( let request_pubkey = withdrawal_request.validator_pubkey # Verify pubkey exists - index = findValidatorIndex(state, request_pubkey).valueOr: + index = findValidatorIndex( + state.validators.asSeq, bucketSortedValidators, + request_pubkey).valueOr: return validator = state.validators.item(index) @@ -591,6 +569,7 @@ func process_withdrawal_request*( # https://github.com/ethereum/consensus-specs/blob/v1.5.0-alpha.3/specs/electra/beacon-chain.md#new-process_consolidation_request proc process_consolidation_request*( cfg: RuntimeConfig, state: var electra.BeaconState, + bucketSortedValidators: BucketSortedValidators, consolidation_request: ConsolidationRequest, cache: var StateCache) = # If the pending consolidations queue is full, consolidation requests are @@ -606,11 +585,14 @@ proc process_consolidation_request*( let # Verify pubkeys exists - source_index = - findValidatorIndex(state, consolidation_request.source_pubkey).valueOr: + source_index = findValidatorIndex( + state.validators.asSeq, bucketSortedValidators, + consolidation_request.source_pubkey).valueOr: return target_index = - findValidatorIndex(state, consolidation_request.target_pubkey).valueOr: + findValidatorIndex( + state.validators.asSeq, bucketSortedValidators, + consolidation_request.target_pubkey).valueOr: return # Verify that source != target, so a consolidation cannot be used as an exit. @@ -698,12 +680,26 @@ proc process_operations( # It costs a full validator set scan to construct these values; only do so if # there will be some kind of exit. - var exit_queue_info = - if body.proposer_slashings.len + body.attester_slashings.len + - body.voluntary_exits.len > 0: - get_state_exit_queue_info(state) - else: - default(ExitQueueInfo) # not used + # TODO Electra doesn't use exit_queue_info, don't calculate + var + exit_queue_info = + if body.proposer_slashings.len + body.attester_slashings.len + + body.voluntary_exits.len > 0: + get_state_exit_queue_info(state) + else: + default(ExitQueueInfo) # not used + bsv_use = + when typeof(body).kind >= ConsensusFork.Electra: + body.deposits.len + body.execution_payload.deposit_requests.len + + body.execution_payload.withdrawal_requests.len + + body.execution_payload.consolidation_requests.len > 0 + else: + body.deposits.len > 0 + bsv = + if bsv_use: + sortValidatorBuckets(state.validators.asSeq) + else: + nil # this is a logic error, effectively assert for op in body.proposer_slashings: let (proposer_slashing_reward, new_exit_queue_info) = @@ -718,10 +714,8 @@ proc process_operations( for op in body.attestations: operations_rewards.attestations += ? process_attestation(state, op, flags, base_reward_per_increment, cache) - if body.deposits.len > 0: - let bloom_filter = constructBloomFilter(state.validators.asSeq) - for op in body.deposits: - ? process_deposit(cfg, state, bloom_filter[], op, flags) + for op in body.deposits: + ? process_deposit(cfg, state, bsv[], op, flags) for op in body.voluntary_exits: exit_queue_info = ? process_voluntary_exit( cfg, state, op, flags, exit_queue_info, cache) @@ -731,15 +725,13 @@ proc process_operations( when typeof(body).kind >= ConsensusFork.Electra: for op in body.execution_payload.deposit_requests: - debugComment "combine with previous Bloom filter construction" - let bloom_filter = constructBloomFilter(state.validators.asSeq) - ? process_deposit_request(cfg, state, bloom_filter[], op, {}) + ? process_deposit_request(cfg, state, bsv[], op, {}) for op in body.execution_payload.withdrawal_requests: # [New in Electra:EIP7002:7251] - process_withdrawal_request(cfg, state, op, cache) + process_withdrawal_request(cfg, state, bsv[], op, cache) for op in body.execution_payload.consolidation_requests: # [New in Electra:EIP7251] - process_consolidation_request(cfg, state, op, cache) + process_consolidation_request(cfg, state, bsv[], op, cache) ok(operations_rewards) diff --git a/beacon_chain/validator_bucket_sort.nim b/beacon_chain/validator_bucket_sort.nim new file mode 100644 index 000000000..2db9cbe2e --- /dev/null +++ b/beacon_chain/validator_bucket_sort.nim @@ -0,0 +1,90 @@ +# beacon_chain +# Copyright (c) 2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} + +import std/typetraits +import "."/spec/crypto +from "."/spec/datatypes/base import Validator, ValidatorIndex, pubkey, `==` + +const + BUCKET_BITS = 9 # >= 13 gets slow to construct + NUM_BUCKETS = 1 shl BUCKET_BITS + +type + # `newSeqUninitialized` requires its type to be SomeNumber + IntValidatorIndex = distinctBase ValidatorIndex + + BucketSortedValidators* = object + bucketSorted*: seq[IntValidatorIndex] + bucketUpperBounds: array[NUM_BUCKETS, uint] # avoids over/underflow checks + extraItems*: seq[ValidatorIndex] + +template getBucketNumber(h: ValidatorPubKey): uint = + # This assumes https://en.wikipedia.org/wiki/Avalanche_effect for uniform + # distribution across pubkeys. ValidatorPubKey specifically satisfies this + # criterion. If required, can look at more input bytes, but ultimately it + # doesn't affect correctness, only speed. + + # Otherwise need more than 2 bytes of input + static: doAssert BUCKET_BITS <= 16 + + const BUCKET_MASK = (NUM_BUCKETS - 1) + ((h.blob[0] * 256 + h.blob[1]) and BUCKET_MASK) + +func sortValidatorBuckets*(validators: openArray[Validator]): + ref BucketSortedValidators {.noinline.} = + var bucketSizes: array[NUM_BUCKETS, uint] + for validator in validators: + inc bucketSizes[getBucketNumber(validator.pubkey)] + + var + bucketInsertPositions: array[NUM_BUCKETS, uint] + accum: uint + for i, s in bucketSizes: + accum += s + bucketInsertPositions[i] = accum + doAssert accum == validators.len.uint + let res = (ref BucketSortedValidators)( + bucketSorted: newSeqUninitialized[IntValidatorIndex](validators.len), + bucketUpperBounds: bucketInsertPositions) + + for i, validator in validators: + let insertPos = + addr bucketInsertPositions[getBucketNumber(validator.pubkey)] + dec insertPos[] + res.bucketSorted[insertPos[]] = i.IntValidatorIndex + + doAssert bucketInsertPositions[0] == 0 + for i in 1 ..< NUM_BUCKETS: + doAssert res.bucketUpperBounds[i - 1] == bucketInsertPositions[i] + + res + +func add*( + bucketSortedValidators: var BucketSortedValidators, + validatorIndex: ValidatorIndex) = + bucketSortedValidators.extraItems.add validatorIndex + +func findValidatorIndex*( + validators: openArray[Validator], bsv: BucketSortedValidators, + pubkey: ValidatorPubKey): Opt[ValidatorIndex] = + for validatorIndex in bsv.extraItems: + if validators[validatorIndex.distinctBase].pubkey == pubkey: + return Opt.some validatorIndex.ValidatorIndex + let + bucketNumber = getBucketNumber(pubkey) + lowerBounds = + if bucketNumber == 0: + 0'u + else: + bsv.bucketUpperBounds[bucketNumber - 1] + + for i in lowerBounds ..< bsv.bucketUpperBounds[bucketNumber]: + if validators[bsv.bucketSorted[i]].pubkey == pubkey: + return Opt.some bsv.bucketSorted[i].ValidatorIndex + Opt.none ValidatorIndex diff --git a/ncli/ncli_common.nim b/ncli/ncli_common.nim index e4d9b5e0f..1565e0a12 100644 --- a/ncli/ncli_common.nim +++ b/ncli/ncli_common.nim @@ -406,6 +406,9 @@ func collectFromAttestations( rewardsAndPenalties[index].inclusion_delay = some(inclusionDelay.uint64) +from ".."/beacon_chain/validator_bucket_sort import + findValidatorIndex, sortValidatorBuckets + proc collectFromDeposits( rewardsAndPenalties: var seq[RewardsAndPenalties], forkedState: ForkedHashedBeaconState, @@ -414,9 +417,12 @@ proc collectFromDeposits( cfg: RuntimeConfig) = withStateAndBlck(forkedState, forkedBlock): for deposit in forkyBlck.message.body.deposits: - let pubkey = deposit.data.pubkey - let amount = deposit.data.amount - var index = findValidatorIndex(forkyState.data, pubkey) + let + pubkey = deposit.data.pubkey + amount = deposit.data.amount + var index = findValidatorIndex( + forkyState.data.validators.asSeq, sortValidatorBuckets( + forkyState.data.validators.asSeq)[], pubkey) if index.isNone: if pubkey in pubkeyToIndex: try: diff --git a/nfuzz/libnfuzz.nim b/nfuzz/libnfuzz.nim index 102ee1b5b..1fdd4e4c3 100644 --- a/nfuzz/libnfuzz.nim +++ b/nfuzz/libnfuzz.nim @@ -141,14 +141,14 @@ func nfuzz_block_header(input: openArray[byte], xoutput: ptr byte, decodeAndProcess(BlockHeaderInput): process_block_header(data.state, data.beaconBlock.message, flags, cache).isOk -from ".."/beacon_chain/bloomfilter import constructBloomFilter +from ".."/beacon_chain/validator_bucket_sort import sortValidatorBuckets proc nfuzz_deposit(input: openArray[byte], xoutput: ptr byte, xoutput_size: ptr uint, disable_bls: bool): bool {.exportc, raises: [FuzzCrashError].} = decodeAndProcess(DepositInput): process_deposit( getRuntimeConfig(some "mainnet"), data.state, - constructBloomFilter(data.state.validators.asSeq)[], data.deposit, + sortValidatorBuckets(data.state.validators.asSeq)[], data.deposit, flags).isOk proc nfuzz_proposer_slashing(input: openArray[byte], xoutput: ptr byte, diff --git a/tests/all_tests.nim b/tests/all_tests.nim index 18f698da6..46a861c16 100644 --- a/tests/all_tests.nim +++ b/tests/all_tests.nim @@ -21,7 +21,6 @@ import # Unit test ./test_block_dag, ./test_block_processor, ./test_block_quarantine, - ./test_bloom_filter, ./test_conf, ./test_datatypes, ./test_deposit_snapshots, @@ -51,6 +50,7 @@ import # Unit test ./test_sync_committee_pool, ./test_sync_manager, ./test_toblindedblock, + ./test_validator_bucket_sort, ./test_validator_change_pool, ./test_validator_pool, ./test_zero_signature, @@ -64,4 +64,4 @@ import # Unit test when not defined(windows): import ./test_keymanager_api -summarizeLongTests("AllTests") +summarizeLongTests("AllTests") \ No newline at end of file diff --git a/tests/consensus_spec/altair/test_fixture_operations.nim b/tests/consensus_spec/altair/test_fixture_operations.nim index 8ca1b4f6c..4bd39a28e 100644 --- a/tests/consensus_spec/altair/test_fixture_operations.nim +++ b/tests/consensus_spec/altair/test_fixture_operations.nim @@ -114,7 +114,8 @@ suite baseDescription & "Block Header " & preset(): runTest[altair.BeaconBlock, typeof applyBlockHeader]( OpBlockHeaderDir, suiteName, "Block Header", "block", applyBlockHeader, path) -from ".."/".."/".."/beacon_chain/bloomfilter import constructBloomFilter +from ".."/".."/".."/beacon_chain/validator_bucket_sort import + sortValidatorBuckets suite baseDescription & "Deposit " & preset(): proc applyDeposit( @@ -122,7 +123,7 @@ suite baseDescription & "Deposit " & preset(): Result[void, cstring] = process_deposit( defaultRuntimeConfig, preState, - constructBloomFilter(preState.validators.asSeq)[], deposit, {}) + sortValidatorBuckets(preState.validators.asSeq)[], deposit, {}) for path in walkTests(OpDepositsDir): runTest[Deposit, typeof applyDeposit]( @@ -173,4 +174,4 @@ suite baseDescription & "Voluntary Exit " & preset(): for path in walkTests(OpVoluntaryExitDir): runTest[SignedVoluntaryExit, typeof applyVoluntaryExit]( OpVoluntaryExitDir, suiteName, "Voluntary Exit", "voluntary_exit", - applyVoluntaryExit, path) + applyVoluntaryExit, path) \ No newline at end of file diff --git a/tests/consensus_spec/bellatrix/test_fixture_operations.nim b/tests/consensus_spec/bellatrix/test_fixture_operations.nim index 7ec841519..f4187c295 100644 --- a/tests/consensus_spec/bellatrix/test_fixture_operations.nim +++ b/tests/consensus_spec/bellatrix/test_fixture_operations.nim @@ -121,7 +121,8 @@ suite baseDescription & "Block Header " & preset(): OpBlockHeaderDir, suiteName, "Block Header", "block", applyBlockHeader, path) -from ".."/".."/".."/beacon_chain/bloomfilter import constructBloomFilter +from ".."/".."/".."/beacon_chain/validator_bucket_sort import + sortValidatorBuckets suite baseDescription & "Deposit " & preset(): proc applyDeposit( @@ -129,7 +130,7 @@ suite baseDescription & "Deposit " & preset(): Result[void, cstring] = process_deposit( defaultRuntimeConfig, preState, - constructBloomFilter(preState.validators.asSeq)[], deposit, {}) + sortValidatorBuckets(preState.validators.asSeq)[], deposit, {}) for path in walkTests(OpDepositsDir): runTest[Deposit, typeof applyDeposit]( @@ -198,4 +199,4 @@ suite baseDescription & "Voluntary Exit " & preset(): for path in walkTests(OpVoluntaryExitDir): runTest[SignedVoluntaryExit, typeof applyVoluntaryExit]( OpVoluntaryExitDir, suiteName, "Voluntary Exit", "voluntary_exit", - applyVoluntaryExit, path) + applyVoluntaryExit, path) \ No newline at end of file diff --git a/tests/consensus_spec/capella/test_fixture_operations.nim b/tests/consensus_spec/capella/test_fixture_operations.nim index 22198cb46..47f9b751d 100644 --- a/tests/consensus_spec/capella/test_fixture_operations.nim +++ b/tests/consensus_spec/capella/test_fixture_operations.nim @@ -138,7 +138,8 @@ suite baseDescription & "BLS to execution change " & preset(): OpBlsToExecutionChangeDir, suiteName, "BLS to execution change", "address_change", applyBlsToExecutionChange, path) -from ".."/".."/".."/beacon_chain/bloomfilter import constructBloomFilter +from ".."/".."/".."/beacon_chain/validator_bucket_sort import + sortValidatorBuckets suite baseDescription & "Deposit " & preset(): func applyDeposit( @@ -146,7 +147,7 @@ suite baseDescription & "Deposit " & preset(): Result[void, cstring] = process_deposit( defaultRuntimeConfig, preState, - constructBloomFilter(preState.validators.asSeq)[], deposit, {}) + sortValidatorBuckets(preState.validators.asSeq)[], deposit, {}) for path in walkTests(OpDepositsDir): runTest[Deposit, typeof applyDeposit]( @@ -226,4 +227,4 @@ suite baseDescription & "Withdrawals " & preset(): for path in walkTests(OpWithdrawalsDir): runTest[capella.ExecutionPayload, typeof applyWithdrawals]( OpWithdrawalsDir, suiteName, "Withdrawals", "execution_payload", - applyWithdrawals, path) + applyWithdrawals, path) \ No newline at end of file diff --git a/tests/consensus_spec/deneb/test_fixture_operations.nim b/tests/consensus_spec/deneb/test_fixture_operations.nim index 78dad933b..56aeb9768 100644 --- a/tests/consensus_spec/deneb/test_fixture_operations.nim +++ b/tests/consensus_spec/deneb/test_fixture_operations.nim @@ -141,7 +141,8 @@ suite baseDescription & "BLS to execution change " & preset(): OpBlsToExecutionChangeDir, suiteName, "BLS to execution change", "address_change", applyBlsToExecutionChange, path) -from ".."/".."/".."/beacon_chain/bloomfilter import constructBloomFilter +from ".."/".."/".."/beacon_chain/validator_bucket_sort import + sortValidatorBuckets suite baseDescription & "Deposit " & preset(): func applyDeposit( @@ -149,7 +150,7 @@ suite baseDescription & "Deposit " & preset(): Result[void, cstring] = process_deposit( defaultRuntimeConfig, preState, - constructBloomFilter(preState.validators.asSeq)[], deposit, {}) + sortValidatorBuckets(preState.validators.asSeq)[], deposit, {}) for path in walkTests(OpDepositsDir): runTest[Deposit, typeof applyDeposit]( @@ -228,4 +229,4 @@ suite baseDescription & "Withdrawals " & preset(): for path in walkTests(OpWithdrawalsDir): runTest[deneb.ExecutionPayload, typeof applyWithdrawals]( OpWithdrawalsDir, suiteName, "Withdrawals", "execution_payload", - applyWithdrawals, path) + applyWithdrawals, path) \ No newline at end of file diff --git a/tests/consensus_spec/electra/test_fixture_operations.nim b/tests/consensus_spec/electra/test_fixture_operations.nim index d3974d56b..a7b21d72c 100644 --- a/tests/consensus_spec/electra/test_fixture_operations.nim +++ b/tests/consensus_spec/electra/test_fixture_operations.nim @@ -148,13 +148,18 @@ suite baseDescription & "BLS to execution change " & preset(): OpBlsToExecutionChangeDir, suiteName, "BLS to execution change", "address_change", applyBlsToExecutionChange, path) +from ".."/".."/".."/beacon_chain/validator_bucket_sort import + sortValidatorBuckets + suite baseDescription & "Consolidation Request " & preset(): proc applyConsolidationRequest( preState: var electra.BeaconState, consolidation_request: ConsolidationRequest): Result[void, cstring] = var cache: StateCache process_consolidation_request( - defaultRuntimeConfig, preState, consolidation_request, cache) + defaultRuntimeConfig, preState, + sortValidatorBuckets(preState.validators.asSeq)[], + consolidation_request, cache) ok() for path in walkTests(OpConsolidationRequestDir): @@ -162,15 +167,13 @@ suite baseDescription & "Consolidation Request " & preset(): OpConsolidationRequestDir, suiteName, "Consolidation Request", "consolidation_request", applyConsolidationRequest, path) -from ".."/".."/".."/beacon_chain/bloomfilter import constructBloomFilter - suite baseDescription & "Deposit " & preset(): func applyDeposit( preState: var electra.BeaconState, deposit: Deposit): Result[void, cstring] = process_deposit( defaultRuntimeConfig, preState, - constructBloomFilter(preState.validators.asSeq)[], deposit, {}) + sortValidatorBuckets(preState.validators.asSeq)[], deposit, {}) for path in walkTests(OpDepositsDir): runTest[Deposit, typeof applyDeposit]( @@ -182,7 +185,7 @@ suite baseDescription & "Deposit Request " & preset(): Result[void, cstring] = process_deposit_request( defaultRuntimeConfig, preState, - constructBloomFilter(preState.validators.asSeq)[], depositRequest, {}) + sortValidatorBuckets(preState.validators.asSeq)[], depositRequest, {}) for path in walkTests(OpDepositRequestDir): runTest[DepositRequest, typeof applyDepositRequest]( @@ -212,7 +215,9 @@ suite baseDescription & "Withdrawal Request " & preset(): Result[void, cstring] = var cache: StateCache process_withdrawal_request( - defaultRuntimeConfig, preState, withdrawalRequest, cache) + defaultRuntimeConfig, preState, + sortValidatorBuckets(preState.validators.asSeq)[], withdrawalRequest, + cache) ok() for path in walkTests(OpWithdrawalRequestDir): @@ -276,4 +281,4 @@ suite baseDescription & "Withdrawals " & preset(): for path in walkTests(OpWithdrawalsDir): runTest[electra.ExecutionPayload, typeof applyWithdrawals]( OpWithdrawalsDir, suiteName, "Withdrawals", "execution_payload", - applyWithdrawals, path) + applyWithdrawals, path) \ No newline at end of file diff --git a/tests/consensus_spec/phase0/test_fixture_operations.nim b/tests/consensus_spec/phase0/test_fixture_operations.nim index 42195af5a..329666902 100644 --- a/tests/consensus_spec/phase0/test_fixture_operations.nim +++ b/tests/consensus_spec/phase0/test_fixture_operations.nim @@ -106,7 +106,8 @@ suite baseDescription & "Block Header " & preset(): OpBlockHeaderDir, suiteName, "Block Header", "block", applyBlockHeader, path) -from ".."/".."/".."/beacon_chain/bloomfilter import constructBloomFilter +from ".."/".."/".."/beacon_chain/validator_bucket_sort import + sortValidatorBuckets suite baseDescription & "Deposit " & preset(): proc applyDeposit( @@ -114,7 +115,7 @@ suite baseDescription & "Deposit " & preset(): Result[void, cstring] = process_deposit( defaultRuntimeConfig, preState, - constructBloomFilter(preState.validators.asSeq)[], deposit, {}) + sortValidatorBuckets(preState.validators.asSeq)[], deposit, {}) for path in walkTests(OpDepositsDir): runTest[Deposit, typeof applyDeposit]( @@ -150,4 +151,4 @@ suite baseDescription & "Voluntary Exit " & preset(): for path in walkTests(OpVoluntaryExitDir): runTest[SignedVoluntaryExit, typeof applyVoluntaryExit]( OpVoluntaryExitDir, suiteName, "Voluntary Exit", "voluntary_exit", - applyVoluntaryExit, path) + applyVoluntaryExit, path) \ No newline at end of file diff --git a/tests/test_bloom_filter.nim b/tests/test_bloom_filter.nim deleted file mode 100644 index 46c00ccfa..000000000 --- a/tests/test_bloom_filter.nim +++ /dev/null @@ -1,147 +0,0 @@ -# beacon_chain -# Copyright (c) 2024 Status Research & Development GmbH -# Licensed and distributed under either of -# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). -# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). -# at your option. This file may not be copied, modified, or distributed except according to those terms. - -{.push raises: [].} -{.used.} - -import ".."/beacon_chain/spec/crypto, unittest2 - -from std/sequtils import mapIt -from ".."/beacon_chain/bloomfilter import - constructBloomFilter, incl, mightContain -from ".."/beacon_chain/spec/datatypes/base import - HashedValidatorPubKey, HashedValidatorPubKeyItem, Validator, fromHex, pubkey -from ".."/beacon_chain/spec/eth2_merkleization import hash_tree_root - -let pubkeys = [ - ValidatorPubKey.fromHex("0xd52edb450c9fdad41ce16d724be7b986a5422f8a791b68a370ef86045a85147cf8f7a6342034958d46a136965b622c48"), - ValidatorPubKey.fromHex("0x6f343f3c55183fc6c980e7597ac47c14b59322e22be9109e7ad8412f5b0e5c918b4e6dd60e5b98eb8d2501a94b2fb022"), - ValidatorPubKey.fromHex("0x5e40d512d91a27aa60e95fa10acb60a8a5dc6d85f2238e6418bfd4ebf44215270301f9e15564dde2c2b628fe80e7f970"), - ValidatorPubKey.fromHex("0x4ae23aea68bfd30022d4efdde1b4428f23317a70fb6df716dc16ccde96b74174c2f8cd18237bdb7ae900acbaba8cad70"), - ValidatorPubKey.fromHex("0x032fc41fa7fc1a44a1f38d73a3465974c2048bb347a9fcb261b93fc6581009d7c9870f0e1a21d619069d5250456cd5ca"), - ValidatorPubKey.fromHex("0x8cea40c0986bc0dc51b664e846a08948112987903b6ffe462b77f092dc43e752dfefaad738810c43364b2f2ed24a5988"), - ValidatorPubKey.fromHex("0xc663a799c732d544a835251935fc5be18eb365806279863877ff2f9308779106816a48be235b4b5d9dcaf42bdf1119f7"), - ValidatorPubKey.fromHex("0xc5682345f202d59614089a6fd5c2375adf8e40316bb69114474f1861c9a6791cc512c0133860353a4bb35d659f3fcd14"), - ValidatorPubKey.fromHex("0x593c3b4d962ff759945f70afa98d3d653fb4c73a2808a4f30472d972cdfd12df7535ba5ba88f3c5e8a59ff844129949f"), - ValidatorPubKey.fromHex("0xabc272512d7a861c0bc190c23cdef8d4d6b9b159d9f53aaf8834c8f521edf416b850d6c14b4c040bac7ceaa1be117e98"), - ValidatorPubKey.fromHex("0xd6dc377e866b762ab63dc2155be71bf24624855e255332dc48a175a9024e71057ad4ad351d7b5aeee944afaaff5d4e1b"), - ValidatorPubKey.fromHex("0x9af21f5d70846185023f70f7841f2f6323c27307c3e54025f103ba359c856b76d3c06f0a09b4669e4838187805253467"), - ValidatorPubKey.fromHex("0x92312221300b0707c401d3163f951babaeb4121fa7222dafebba8b8cf91928567477b4b2c249af446a759ef13d990a0c"), - ValidatorPubKey.fromHex("0x37c2731f409eafdb4bb5a1722e33cc39ab8dcf87eb7b4702aca0dcfdceea15002c1b697124eb6f1f83bd807cafb0ff43"), - ValidatorPubKey.fromHex("0xac72cfe3b2a0c549f608746fd0c3daa7195c42e05157f8d8b10bd84b1d04bff763eb6bf74620be8bcdba0ea4704630ee"), - ValidatorPubKey.fromHex("0x6cab2ab1fd15489aae21becc2cfb8923513bacce9d9773c3ad35ef7535a6e92d3a78de4d103e2ed88a818f872de331f2"), - ValidatorPubKey.fromHex("0x99138fe703da75af5571e3994e7c0b6bba06cb2a4a4978e4b41e52e06af7c1c928105bb5fae878d16934529c96883e97"), - ValidatorPubKey.fromHex("0x850c61b9bf24be2470fe0b1ead466d9b93ea4b4d41980f2f6c82eef9b526d68bf6be613b4e7653b79267829a4107dd30"), - ValidatorPubKey.fromHex("0x310ddff78f82b2ea039f6077b099f4e8e148da97d35a14140cdf5754db933034d15a58085ff91522e2722504a6ebdc87"), - ValidatorPubKey.fromHex("0x331103905b6cc0da6ef1fc2e10cb6c9feed110a5a09fed5f32f56416ea814e80961fdf81455a6483de18c40e1f3bb718"), - ValidatorPubKey.fromHex("0x8f4a32c968cb197581a3c4cec214d33736026997d1a4dc9538c932b3d859dd0547a7a06a08a9115c2c2a4fdfccaa07d2"), - ValidatorPubKey.fromHex("0xda87a0a9a300057c1f4a196f9e8947a1f461aca3be84799ac9a187c4ecb0f6450cc15e64d30b30da4f5cf2848808b9ab"), - ValidatorPubKey.fromHex("0x91e197089e1a351f0f6b1d4777c464edffac62067162133c01185074d520cbefd4e661d978cf04f9832804cb636e7a5f"), - ValidatorPubKey.fromHex("0xf0e76be22bf4afd4ea3730ef7dd0156b777e2835d828deee887881263affa33bf4685ad18fa05d09e87481a4c89c345c"), - ValidatorPubKey.fromHex("0x4a0276deca3b176cd6fe0b648f0fc418568c0c9d29d607e74e02c17852b72e636e681f4be63b0b1ad842db3efe0518c2"), - ValidatorPubKey.fromHex("0x7ad942fe106ee88c214bd5e34078b2c98849ba594a4e266a8548c1b5e44bd151135fa5a720323927c142af19fd1e74b1"), - ValidatorPubKey.fromHex("0x0648a3a4f9cf10e8f8881902549e0b7c6b207e72d5498e54503e1497ccfc03954a7440dfa0cd5ba62f80234bd99733ca"), - ValidatorPubKey.fromHex("0x5d569974f21599857609ec27e11cd2b9c007209790fe36e0cc5ff1bef0c83c07eddc84602ae04a3b803b158fa8d8a7df"), - ValidatorPubKey.fromHex("0x63290edbc38bfa204b7fd4b3fba3f677f00a54897b4c62c83ff5a1d0a905f64d2ea73ab9fa903d86c3ac8e5c91f66cc2"), - ValidatorPubKey.fromHex("0xc56363e2f8a19dcb1c9fa0b446b9c2e6a93218250df814da9566c4ceaeb116a4d60031ec60b89c23e0e911dccc301e34"), - ValidatorPubKey.fromHex("0x68c143f8c1cf0dc47345526bfd5123ed31edcbf393673352fe948107f5317ddcf8934814657879da7a1ec5782d13fdc4"), - ValidatorPubKey.fromHex("0x6e1c7d1ca0056d721a94cda0a776b68d447b1706882e04ed7ca7356d61d7d08c9c2aaf782e9c3f0c4c6e4758ca6c9228"), - ValidatorPubKey.fromHex("0x12d410ee83662b4506546e912ada2e0273f27279fdc46565d0c862e262bdbe98f91466a5bfa4e65660fd8e5a4da28543"), - ValidatorPubKey.fromHex("0x039b3ebfcc2d6f181b40da2b63d94406c440f2c32547e69560bb137a295886c3e82b7ac5aa18e14bfe080b805ae75197"), - ValidatorPubKey.fromHex("0x02875a3d83a806329b612096329959eec1a2300d9740a2c94d030dc5c99c6c0c62bd5f832b615d36cc165bc304e7a892"), - ValidatorPubKey.fromHex("0xfc0acd4ca1e1ea234b39219af5c899a743332e33a17be2dcb10bfed72e4d437fd2693ac1ae1dcec2d189a9689b0b53ff"), - ValidatorPubKey.fromHex("0x8104b3b199bf0261b1584fe269e5599266bd60cbd201205565e139fbe4a9577f48f71cebae7f7cf434cf07f66cc51ec9"), - ValidatorPubKey.fromHex("0xcfe998a8989f5318aee192e185e87a51b96aeec479d37f00cdcfafe534f514c316a10c4ba311c076cae6b192386dc25a"), - ValidatorPubKey.fromHex("0x44d7bcaebb2da8001982540c8917da5ff950750d90e5b788f2c35262b58efca64dfe3df46793380016a9d24b521c3920"), - ValidatorPubKey.fromHex("0x2b7fd53635b1effa086d6db933b65bfbca85160ed731fa8b5c77a8b726b4c5b61ff56d88d57f3e4fece8c593df18f2b3"), - ValidatorPubKey.fromHex("0x642e56b532e08e4cb75d619ed3b360ad1971584e338638b7d5716672922e513465c3fb13d26d381e7b21ffe9bc8e428f"), - ValidatorPubKey.fromHex("0x61820ec30590c9e75b06b0cc454686067fc6db1d329814aaf1a31e3e3defe50f41ee15c106e3602c4931e131032787db"), - ValidatorPubKey.fromHex("0xdc41f2c1504c90f44ba32b7e9d8e069d9c788a125f45df163c65c56cf22f5823e7614b2fcd5cec7c14a276b67e0fa7b8"), - ValidatorPubKey.fromHex("0x079d59adc0ac14e2c7397a23c3debcb080d1378ad4ac6a091daeb12f1d134c063ce4629bdf0880172017b81bed0064ec"), - ValidatorPubKey.fromHex("0x41e0b5b8befce0add67f48a9b485307105e3772aae012777c6afa62304f67a7407dd0c16b791754076549eba2b7a18a8"), - ValidatorPubKey.fromHex("0xd36e7623ae93544eaa5868e50936797bddffb2b3b66728b38f0c479f1640c60e82ad887b960e6c9340526da8a030f5b2"), - ValidatorPubKey.fromHex("0x8986816ba54e777b2c6045a805b11c08bb1f64898a6786428da9efc2ae466cb940fa3c11feacfdeeba87df9b3ce3e93f"), - ValidatorPubKey.fromHex("0x5ea844f61fe1710c19cb67e5daec1c3ba0fc203ab23598b1c9cfae6f4ab9d9f127d50d0b9cebf64d7650f66c06ca5596"), - ValidatorPubKey.fromHex("0x3e77eef77d7573362dffd75800d7554ad41f4349b3a2ab72d6fe031bf3c42bf283f985b933ac142de581079371018fdc"), - ValidatorPubKey.fromHex("0xa848afaf6d44d43e2f072bf3cf82e1ae6a8c63cda627c12d95a43e6ac4f20b8a9213a723d642c95ae2bd66bccadb8467"), - ValidatorPubKey.fromHex("0xb0b1b8582a84cbc5f43585c9d2e2b9d34f0788977f5004d6e21b12bfd5cd7165d72fba0da182f13aa44af63f9045da3e"), - ValidatorPubKey.fromHex("0x4f5517fe02d94b1eeee0a294b4f7d6064f8a3eb3fd6f31801ab7545be1dc290f26972515b23018b23efa9a812f648b6b"), - ValidatorPubKey.fromHex("0xa0f040547549deccd5cdc3a0a3a91974754fdc8177763adfc25ffb7704f8ca5e83985db3f276fadb1c113fb279720a05"), - ValidatorPubKey.fromHex("0x7dd6ae00b240244b0e49cf7538a5021e6725d3b87b909e797c7d9c6947c3b29353ff61c128ad36db66b77f197308ba04"), - ValidatorPubKey.fromHex("0xdc824ba613c5ddf2c112f0ca3bb91e6d7bfcbfd340b1e611183b8bf8c4cc37d1b843909f2c9db8353de6938834516fa2"), - ValidatorPubKey.fromHex("0xb085822d9549b0b674591015525f0846ec00ef3ff52b1107592285d0a75b757708a54fcfe5655f28473c33ae4d43ee5c"), - ValidatorPubKey.fromHex("0xab704b4be6cbbbe0f9176fd3dccbf2c0272e4f42538d5f4844a288820179f7c799d051c501e78ee3848484e1818d8456"), - ValidatorPubKey.fromHex("0x12c3c3fa284bd55ebbe82abce576c104929a909e9d78eba2f595ce42822ffe52c427ad61923f48107b1639e4bd99a45b"), - ValidatorPubKey.fromHex("0x64c86e12cdc8091c0b0e317abc073a71c96df04e1fb2235219a1289d3ce62b323fc1a226f0b298ee5596bbebabdacaf5"), - ValidatorPubKey.fromHex("0x1d5cc7e50da341a6f6931dc9fb4df6a37d21545281b9fdc2836182e2f45ff2a2a6e9181ab5d4893125fea6495fe68dd3"), - ValidatorPubKey.fromHex("0x923573206c1b1a75716339eb61f489b10d5811a280dd15333f980374ca63664741e16d911f8372ff74714ec79662683f"), - ValidatorPubKey.fromHex("0x7c1fe9a7ab8da368228a27f575cbb36aa9ce2e68d60c336184f02b985b5c13a7d09cbe315895a1da5f1f86d713f94417"), - ValidatorPubKey.fromHex("0xbb85e9cdac2db9a2dda61480082f3ed0f683db798219cdbfadac846c7b374f90a8c6784c95b53676b631152077619ee5"), - ValidatorPubKey.fromHex("0x58db99741e4c904ec1444a9c23c287eeea88de3c647c9dd9ed45e8230b7ed0bf080d546ae4597af148b69809df07e73c"), - ValidatorPubKey.fromHex("0x2208988a10feef0f7ec1550e8ef8c14c786de0bd647e5b3d10d3b884c8521af0ce59ba1a8583afe888b9348d2e1ed7d5"), - ValidatorPubKey.fromHex("0xd11cd69262896cf2a19a52928b7fcba8cd1c1661d0c938ffbfb4482283f53b44435af5695ce10fddc9315393aeda57ef"), - ValidatorPubKey.fromHex("0x4a568216203673c3f895529c194c2ca172d613e8f866dd9ee5e8db9b5b681942c7b5634c2349689a6753e1d1113d062e"), - ValidatorPubKey.fromHex("0x7ceb8add4aebaf802c3e8b37f85076a6de8c6d7007dcb92fa7b4da028a571f9dae41338b8d3f2446db4335ffbff7f083"), - ValidatorPubKey.fromHex("0xfda68482093ff5780855a139008ba695a1bd74864cb4ff72451caa5a46f8db497b44baecc93ead6aacd34c9ac92522d4"), - ValidatorPubKey.fromHex("0x8483c152bf17da7df9f3e7102d2fdd143b7649a95920263c8231ce6e80f01a849ae62064f2d03d6dcb89024d07ef9f33"), - ValidatorPubKey.fromHex("0x33ea02799800edf1c7660f1acf923f33913f2eaa89944c3b8ca4e44a2d061a1c6e4286ca92251bc0f3b11c535824aa0e"), - ValidatorPubKey.fromHex("0x46e3fdc0b5b6df3147a95ccfdfe66373bdbf96e6d5eed7306428f986778dd3b9eecb0bc5e568213b0b3faee7ce6caa79"), - ValidatorPubKey.fromHex("0xac9df2f76111be4c822a91d24a85291f55ed4ae4c574803781018360f83cc395fee9a3e56d92fc34d2f74f4dcd81c19d"), - ValidatorPubKey.fromHex("0xe6724c500b1573fee191980bdf4d8e30086bc2f2460ac775d9ceec553d4870f314fae83d04b9d9f17dc1bec64e1b5260"), - ValidatorPubKey.fromHex("0xb45d08842d2721b18d17209081b5b95ed2b9198c0dd47d16117834e1b96913071f5afe5abe53206a10103baeadbc4314"), - ValidatorPubKey.fromHex("0x8badb39dec9b9c348e4833797ac1f7fc84f7bac557d1bed58096144f48b8cda5fd8ddbe21e278f0b6d5c9aed6c90f783"), - ValidatorPubKey.fromHex("0x5fd79ebdc6f58defee05a823c9d793dfdc4b0c43ddbd1eb74c3432f59d069fe026ead5b1c925626ed9f915aee6f91247"), - ValidatorPubKey.fromHex("0x7763334ab10953dea5bffac69dea12eb53f0cd46947f04334d417223040453cfbe0f658d6f1e22a79c09807bdf3ee2c1"), - ValidatorPubKey.fromHex("0xf2df734e8b11d562900079828c2cde7dca86a2d63cf57813c67bab47fc627f1bb773d70015a486a1a2cd09b4a04c1b28"), - ValidatorPubKey.fromHex("0xd0c621f5bb524fb68aa3631b4a0629bf6bc210fe30e237d9caf8bfb476686b82eb8e8460062d187d6e2699ddc8988c0c"), - ValidatorPubKey.fromHex("0x10eb53f3ba6d355e301c785a2f204294c6a63233edee9cc135791815d086c9a8604c0d46baca6abe8c7a58e708e2106a"), - ValidatorPubKey.fromHex("0x4244a5380986232f8fb39f9396be04e6c504c3b1f87e9672d7154d09b97f0fa86cae849aac06b30ce993e00e126cf5b0"), - ValidatorPubKey.fromHex("0x2382850a411c389df2afdd2a03a6196b451893e2674d11e0b8ac6914ffa53c7a1ced201cc1390a3aa1a2879dcdfa143b"), - ValidatorPubKey.fromHex("0xa20189e31ecc6a8c2002a9dec9645aada8f01dbaa6f22f7efcc10e1de109f2528edcbe768f1baf78b8ecba189d70e28b"), - ValidatorPubKey.fromHex("0xd1f4e4ebedcc39544148157f4a5279def61a8dda08c087afbcc85e85f5fe8a244972e26077cfc1820c0c85814adfad6e"), - ValidatorPubKey.fromHex("0xf62d8f1b982babdffcc6616f8b2ac54fac5224c7a1fb66121079b9a521aff4f2ade3cd7aa40baa838e522a927179ac82"), - ValidatorPubKey.fromHex("0x7e0c87bbf88d5762dfca69732bb36525d11a755fde736f28088bc17958cb8d5745a923a56c6c0b4e98c0ffd9623f9816"), - ValidatorPubKey.fromHex("0xbf1d6ae7fd84bee92a4e22bd73b3869402504736ff5af0da6e02181ae2506a248ca4e969a82ea0304a93b6bb68d29435"), - ValidatorPubKey.fromHex("0x8ec4826fcde422ba62d222274fda595cd988d27fa0ffcbc91ab7ace22d2c9617a94ba008064a5f159801dc3b1956d96f"), - ValidatorPubKey.fromHex("0x068bee5a0d17f286962fdf71fe6e9d8b2d05f8203ecf2fbc0672003ec18a53636062dabd430715b8599f1111091417dd"), - ValidatorPubKey.fromHex("0xc0e15eadc90fbf93e2deccdd58cb13b30fea11913ca39c2ee42ddf74201dae1e84553ce8c6818d91658cb8ae97573c24"), - ValidatorPubKey.fromHex("0x5a0e0446883b0a0f09ea42faffc02ebf25407159503f5b430a216a54b1b9a4272765314c267ee2f3be8fe101208a28fd"), - ValidatorPubKey.fromHex("0xc22aa9c85a08126c371c19163c940c459a478a7391cabfb170a352faa30687ef571568d4ad327a6fe69652cd0daa33af"), - ValidatorPubKey.fromHex("0xc53c961a6977d4711914b2852ac231e6dae019ce13555e189bcae94b1786f0bb3b3e8ad173c3f029758ecbc0c0b1c6f0"), - ValidatorPubKey.fromHex("0x925aefdfeaeea3402ddd678a7069c20183fed9a11f7f866215788177ba9ae9d2914874866c2dd78f79f81495ce172352"), - ValidatorPubKey.fromHex("0x4aca00821c817196db75be87cb044f36466c65e5ea3ca90c60353b3927107bdbd8ec0775dfe8c08ea123801f4443d01b"), - ValidatorPubKey.fromHex("0xb84960b4042210498cd2ab478685a1b65e2a4e3bbf2e813440e38f38659def0e5ebe9514316f125634e23ae398fa2458"), - ValidatorPubKey.fromHex("0x3dbee79b334a30be85c82ae64331ab0bd7ce371c2b5cc734212f079209a845d0f45393bbca97ffad203e0af81af4325b"), - ValidatorPubKey.fromHex("0xfd9e33dec3e8ebeeb2ec64297ace2997dc6ecf148d98067cc3aabf2419a2788160c4d670836419672eebd663999ba53b"), - ValidatorPubKey.fromHex("0xdd9de04d992ecd5991ed84567803f2195b9c0cbbf74968e60c2272ba59f741fb07e84eefd970a0507b36ad7e4bd56e7e")] - -suite "ValidatorPubKey Bloom filter": - test "one-shot construction with no false positives/negatives": - var hashedPubkeyItems = mapIt(pubkeys, HashedValidatorPubKeyItem( - key: it.get, root: hash_tree_root(it.get))) - let - hashedPubkeys = mapIt(hashedPubkeyItems, HashedValidatorPubKey( - value: unsafeAddr it)) - validators = mapIt(hashedPubkeys, Validator(pubkeyData: it)) - - let bloomFilter = constructBloomFilter( - validators.toOpenArray(0, validators.len div 2)) - for validator in validators.toOpenArray(0, validators.len div 2): - check: bloomFilter[].mightContain(validator.pubkey) - for validator in validators.toOpenArray( - validators.len div 2 + 1, validators.len - 1): - check: not bloomFilter[].mightContain(validator.pubkey) - - test "incremental construction with no false positives/negatives": - let bloomFilter = constructBloomFilter([]) - for pubkey in pubkeys.toOpenArray(0, pubkeys.len div 2): - incl(bloomFilter[], pubkey.get) - - for pubkey in pubkeys.toOpenArray(0, pubkeys.len div 2): - check: bloomFilter[].mightContain(pubkey.get) - for pubkey in pubkeys.toOpenArray(pubkeys.len div 2 + 1, pubkeys.len - 1): - check: not bloomFilter[].mightContain(pubkey.get) diff --git a/tests/test_validator_bucket_sort.nim b/tests/test_validator_bucket_sort.nim new file mode 100644 index 000000000..d48b42e85 --- /dev/null +++ b/tests/test_validator_bucket_sort.nim @@ -0,0 +1,260 @@ +# beacon_chain +# Copyright (c) 2024 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +{.push raises: [].} +{.used.} + +import std/typetraits +import ".."/beacon_chain/spec/crypto, unittest2 + +from std/sequtils import mapIt +from ".."/beacon_chain/validator_bucket_sort import + BucketSortedValidators, add, findValidatorIndex, sortValidatorBuckets +from ".."/beacon_chain/spec/datatypes/base import + HashedValidatorPubKey, HashedValidatorPubKeyItem, Validator, ValidatorIndex, + fromHex, pubkey, `==` +from ".."/beacon_chain/spec/eth2_merkleization import hash_tree_root + +let pubkeys = [ + ValidatorPubKey.fromHex("0xd52edb450c9fdad41ce16d724be7b986a5422f8a791b68a370ef86045a85147cf8f7a6342034958d46a136965b622c48"), + ValidatorPubKey.fromHex("0x6f343f3c55183fc6c980e7597ac47c14b59322e22be9109e7ad8412f5b0e5c918b4e6dd60e5b98eb8d2501a94b2fb022"), + ValidatorPubKey.fromHex("0x5e40d512d91a27aa60e95fa10acb60a8a5dc6d85f2238e6418bfd4ebf44215270301f9e15564dde2c2b628fe80e7f970"), + ValidatorPubKey.fromHex("0x4ae23aea68bfd30022d4efdde1b4428f23317a70fb6df716dc16ccde96b74174c2f8cd18237bdb7ae900acbaba8cad70"), + ValidatorPubKey.fromHex("0x032fc41fa7fc1a44a1f38d73a3465974c2048bb347a9fcb261b93fc6581009d7c9870f0e1a21d619069d5250456cd5ca"), + ValidatorPubKey.fromHex("0x8cea40c0986bc0dc51b664e846a08948112987903b6ffe462b77f092dc43e752dfefaad738810c43364b2f2ed24a5988"), + ValidatorPubKey.fromHex("0xc663a799c732d544a835251935fc5be18eb365806279863877ff2f9308779106816a48be235b4b5d9dcaf42bdf1119f7"), + ValidatorPubKey.fromHex("0xc5682345f202d59614089a6fd5c2375adf8e40316bb69114474f1861c9a6791cc512c0133860353a4bb35d659f3fcd14"), + ValidatorPubKey.fromHex("0x593c3b4d962ff759945f70afa98d3d653fb4c73a2808a4f30472d972cdfd12df7535ba5ba88f3c5e8a59ff844129949f"), + ValidatorPubKey.fromHex("0xabc272512d7a861c0bc190c23cdef8d4d6b9b159d9f53aaf8834c8f521edf416b850d6c14b4c040bac7ceaa1be117e98"), + ValidatorPubKey.fromHex("0xd6dc377e866b762ab63dc2155be71bf24624855e255332dc48a175a9024e71057ad4ad351d7b5aeee944afaaff5d4e1b"), + ValidatorPubKey.fromHex("0x9af21f5d70846185023f70f7841f2f6323c27307c3e54025f103ba359c856b76d3c06f0a09b4669e4838187805253467"), + ValidatorPubKey.fromHex("0x92312221300b0707c401d3163f951babaeb4121fa7222dafebba8b8cf91928567477b4b2c249af446a759ef13d990a0c"), + ValidatorPubKey.fromHex("0x37c2731f409eafdb4bb5a1722e33cc39ab8dcf87eb7b4702aca0dcfdceea15002c1b697124eb6f1f83bd807cafb0ff43"), + ValidatorPubKey.fromHex("0xac72cfe3b2a0c549f608746fd0c3daa7195c42e05157f8d8b10bd84b1d04bff763eb6bf74620be8bcdba0ea4704630ee"), + ValidatorPubKey.fromHex("0x6cab2ab1fd15489aae21becc2cfb8923513bacce9d9773c3ad35ef7535a6e92d3a78de4d103e2ed88a818f872de331f2"), + ValidatorPubKey.fromHex("0x99138fe703da75af5571e3994e7c0b6bba06cb2a4a4978e4b41e52e06af7c1c928105bb5fae878d16934529c96883e97"), + ValidatorPubKey.fromHex("0x850c61b9bf24be2470fe0b1ead466d9b93ea4b4d41980f2f6c82eef9b526d68bf6be613b4e7653b79267829a4107dd30"), + ValidatorPubKey.fromHex("0x310ddff78f82b2ea039f6077b099f4e8e148da97d35a14140cdf5754db933034d15a58085ff91522e2722504a6ebdc87"), + ValidatorPubKey.fromHex("0x331103905b6cc0da6ef1fc2e10cb6c9feed110a5a09fed5f32f56416ea814e80961fdf81455a6483de18c40e1f3bb718"), + ValidatorPubKey.fromHex("0x8f4a32c968cb197581a3c4cec214d33736026997d1a4dc9538c932b3d859dd0547a7a06a08a9115c2c2a4fdfccaa07d2"), + ValidatorPubKey.fromHex("0xda87a0a9a300057c1f4a196f9e8947a1f461aca3be84799ac9a187c4ecb0f6450cc15e64d30b30da4f5cf2848808b9ab"), + ValidatorPubKey.fromHex("0x91e197089e1a351f0f6b1d4777c464edffac62067162133c01185074d520cbefd4e661d978cf04f9832804cb636e7a5f"), + ValidatorPubKey.fromHex("0xf0e76be22bf4afd4ea3730ef7dd0156b777e2835d828deee887881263affa33bf4685ad18fa05d09e87481a4c89c345c"), + ValidatorPubKey.fromHex("0x4a0276deca3b176cd6fe0b648f0fc418568c0c9d29d607e74e02c17852b72e636e681f4be63b0b1ad842db3efe0518c2"), + ValidatorPubKey.fromHex("0x7ad942fe106ee88c214bd5e34078b2c98849ba594a4e266a8548c1b5e44bd151135fa5a720323927c142af19fd1e74b1"), + ValidatorPubKey.fromHex("0x0648a3a4f9cf10e8f8881902549e0b7c6b207e72d5498e54503e1497ccfc03954a7440dfa0cd5ba62f80234bd99733ca"), + ValidatorPubKey.fromHex("0x5d569974f21599857609ec27e11cd2b9c007209790fe36e0cc5ff1bef0c83c07eddc84602ae04a3b803b158fa8d8a7df"), + ValidatorPubKey.fromHex("0x63290edbc38bfa204b7fd4b3fba3f677f00a54897b4c62c83ff5a1d0a905f64d2ea73ab9fa903d86c3ac8e5c91f66cc2"), + ValidatorPubKey.fromHex("0xc56363e2f8a19dcb1c9fa0b446b9c2e6a93218250df814da9566c4ceaeb116a4d60031ec60b89c23e0e911dccc301e34"), + ValidatorPubKey.fromHex("0x68c143f8c1cf0dc47345526bfd5123ed31edcbf393673352fe948107f5317ddcf8934814657879da7a1ec5782d13fdc4"), + ValidatorPubKey.fromHex("0x6e1c7d1ca0056d721a94cda0a776b68d447b1706882e04ed7ca7356d61d7d08c9c2aaf782e9c3f0c4c6e4758ca6c9228"), + ValidatorPubKey.fromHex("0x12d410ee83662b4506546e912ada2e0273f27279fdc46565d0c862e262bdbe98f91466a5bfa4e65660fd8e5a4da28543"), + ValidatorPubKey.fromHex("0x039b3ebfcc2d6f181b40da2b63d94406c440f2c32547e69560bb137a295886c3e82b7ac5aa18e14bfe080b805ae75197"), + ValidatorPubKey.fromHex("0x02875a3d83a806329b612096329959eec1a2300d9740a2c94d030dc5c99c6c0c62bd5f832b615d36cc165bc304e7a892"), + ValidatorPubKey.fromHex("0xfc0acd4ca1e1ea234b39219af5c899a743332e33a17be2dcb10bfed72e4d437fd2693ac1ae1dcec2d189a9689b0b53ff"), + ValidatorPubKey.fromHex("0x8104b3b199bf0261b1584fe269e5599266bd60cbd201205565e139fbe4a9577f48f71cebae7f7cf434cf07f66cc51ec9"), + ValidatorPubKey.fromHex("0xcfe998a8989f5318aee192e185e87a51b96aeec479d37f00cdcfafe534f514c316a10c4ba311c076cae6b192386dc25a"), + ValidatorPubKey.fromHex("0x44d7bcaebb2da8001982540c8917da5ff950750d90e5b788f2c35262b58efca64dfe3df46793380016a9d24b521c3920"), + ValidatorPubKey.fromHex("0x2b7fd53635b1effa086d6db933b65bfbca85160ed731fa8b5c77a8b726b4c5b61ff56d88d57f3e4fece8c593df18f2b3"), + ValidatorPubKey.fromHex("0x642e56b532e08e4cb75d619ed3b360ad1971584e338638b7d5716672922e513465c3fb13d26d381e7b21ffe9bc8e428f"), + ValidatorPubKey.fromHex("0x61820ec30590c9e75b06b0cc454686067fc6db1d329814aaf1a31e3e3defe50f41ee15c106e3602c4931e131032787db"), + ValidatorPubKey.fromHex("0xdc41f2c1504c90f44ba32b7e9d8e069d9c788a125f45df163c65c56cf22f5823e7614b2fcd5cec7c14a276b67e0fa7b8"), + ValidatorPubKey.fromHex("0x079d59adc0ac14e2c7397a23c3debcb080d1378ad4ac6a091daeb12f1d134c063ce4629bdf0880172017b81bed0064ec"), + ValidatorPubKey.fromHex("0x41e0b5b8befce0add67f48a9b485307105e3772aae012777c6afa62304f67a7407dd0c16b791754076549eba2b7a18a8"), + ValidatorPubKey.fromHex("0xd36e7623ae93544eaa5868e50936797bddffb2b3b66728b38f0c479f1640c60e82ad887b960e6c9340526da8a030f5b2"), + ValidatorPubKey.fromHex("0x8986816ba54e777b2c6045a805b11c08bb1f64898a6786428da9efc2ae466cb940fa3c11feacfdeeba87df9b3ce3e93f"), + ValidatorPubKey.fromHex("0x5ea844f61fe1710c19cb67e5daec1c3ba0fc203ab23598b1c9cfae6f4ab9d9f127d50d0b9cebf64d7650f66c06ca5596"), + ValidatorPubKey.fromHex("0x3e77eef77d7573362dffd75800d7554ad41f4349b3a2ab72d6fe031bf3c42bf283f985b933ac142de581079371018fdc"), + ValidatorPubKey.fromHex("0xa848afaf6d44d43e2f072bf3cf82e1ae6a8c63cda627c12d95a43e6ac4f20b8a9213a723d642c95ae2bd66bccadb8467"), + ValidatorPubKey.fromHex("0xb0b1b8582a84cbc5f43585c9d2e2b9d34f0788977f5004d6e21b12bfd5cd7165d72fba0da182f13aa44af63f9045da3e"), + ValidatorPubKey.fromHex("0x4f5517fe02d94b1eeee0a294b4f7d6064f8a3eb3fd6f31801ab7545be1dc290f26972515b23018b23efa9a812f648b6b"), + ValidatorPubKey.fromHex("0xa0f040547549deccd5cdc3a0a3a91974754fdc8177763adfc25ffb7704f8ca5e83985db3f276fadb1c113fb279720a05"), + ValidatorPubKey.fromHex("0x7dd6ae00b240244b0e49cf7538a5021e6725d3b87b909e797c7d9c6947c3b29353ff61c128ad36db66b77f197308ba04"), + ValidatorPubKey.fromHex("0xdc824ba613c5ddf2c112f0ca3bb91e6d7bfcbfd340b1e611183b8bf8c4cc37d1b843909f2c9db8353de6938834516fa2"), + ValidatorPubKey.fromHex("0xb085822d9549b0b674591015525f0846ec00ef3ff52b1107592285d0a75b757708a54fcfe5655f28473c33ae4d43ee5c"), + ValidatorPubKey.fromHex("0xab704b4be6cbbbe0f9176fd3dccbf2c0272e4f42538d5f4844a288820179f7c799d051c501e78ee3848484e1818d8456"), + ValidatorPubKey.fromHex("0x12c3c3fa284bd55ebbe82abce576c104929a909e9d78eba2f595ce42822ffe52c427ad61923f48107b1639e4bd99a45b"), + ValidatorPubKey.fromHex("0x64c86e12cdc8091c0b0e317abc073a71c96df04e1fb2235219a1289d3ce62b323fc1a226f0b298ee5596bbebabdacaf5"), + ValidatorPubKey.fromHex("0x1d5cc7e50da341a6f6931dc9fb4df6a37d21545281b9fdc2836182e2f45ff2a2a6e9181ab5d4893125fea6495fe68dd3"), + ValidatorPubKey.fromHex("0x923573206c1b1a75716339eb61f489b10d5811a280dd15333f980374ca63664741e16d911f8372ff74714ec79662683f"), + ValidatorPubKey.fromHex("0x7c1fe9a7ab8da368228a27f575cbb36aa9ce2e68d60c336184f02b985b5c13a7d09cbe315895a1da5f1f86d713f94417"), + ValidatorPubKey.fromHex("0xbb85e9cdac2db9a2dda61480082f3ed0f683db798219cdbfadac846c7b374f90a8c6784c95b53676b631152077619ee5"), + ValidatorPubKey.fromHex("0x58db99741e4c904ec1444a9c23c287eeea88de3c647c9dd9ed45e8230b7ed0bf080d546ae4597af148b69809df07e73c"), + ValidatorPubKey.fromHex("0x2208988a10feef0f7ec1550e8ef8c14c786de0bd647e5b3d10d3b884c8521af0ce59ba1a8583afe888b9348d2e1ed7d5"), + ValidatorPubKey.fromHex("0xd11cd69262896cf2a19a52928b7fcba8cd1c1661d0c938ffbfb4482283f53b44435af5695ce10fddc9315393aeda57ef"), + ValidatorPubKey.fromHex("0x4a568216203673c3f895529c194c2ca172d613e8f866dd9ee5e8db9b5b681942c7b5634c2349689a6753e1d1113d062e"), + ValidatorPubKey.fromHex("0x7ceb8add4aebaf802c3e8b37f85076a6de8c6d7007dcb92fa7b4da028a571f9dae41338b8d3f2446db4335ffbff7f083"), + ValidatorPubKey.fromHex("0xfda68482093ff5780855a139008ba695a1bd74864cb4ff72451caa5a46f8db497b44baecc93ead6aacd34c9ac92522d4"), + ValidatorPubKey.fromHex("0x8483c152bf17da7df9f3e7102d2fdd143b7649a95920263c8231ce6e80f01a849ae62064f2d03d6dcb89024d07ef9f33"), + ValidatorPubKey.fromHex("0x33ea02799800edf1c7660f1acf923f33913f2eaa89944c3b8ca4e44a2d061a1c6e4286ca92251bc0f3b11c535824aa0e"), + ValidatorPubKey.fromHex("0x46e3fdc0b5b6df3147a95ccfdfe66373bdbf96e6d5eed7306428f986778dd3b9eecb0bc5e568213b0b3faee7ce6caa79"), + ValidatorPubKey.fromHex("0xac9df2f76111be4c822a91d24a85291f55ed4ae4c574803781018360f83cc395fee9a3e56d92fc34d2f74f4dcd81c19d"), + ValidatorPubKey.fromHex("0xe6724c500b1573fee191980bdf4d8e30086bc2f2460ac775d9ceec553d4870f314fae83d04b9d9f17dc1bec64e1b5260"), + ValidatorPubKey.fromHex("0xb45d08842d2721b18d17209081b5b95ed2b9198c0dd47d16117834e1b96913071f5afe5abe53206a10103baeadbc4314"), + ValidatorPubKey.fromHex("0x8badb39dec9b9c348e4833797ac1f7fc84f7bac557d1bed58096144f48b8cda5fd8ddbe21e278f0b6d5c9aed6c90f783"), + ValidatorPubKey.fromHex("0x5fd79ebdc6f58defee05a823c9d793dfdc4b0c43ddbd1eb74c3432f59d069fe026ead5b1c925626ed9f915aee6f91247"), + ValidatorPubKey.fromHex("0x7763334ab10953dea5bffac69dea12eb53f0cd46947f04334d417223040453cfbe0f658d6f1e22a79c09807bdf3ee2c1"), + ValidatorPubKey.fromHex("0xf2df734e8b11d562900079828c2cde7dca86a2d63cf57813c67bab47fc627f1bb773d70015a486a1a2cd09b4a04c1b28"), + ValidatorPubKey.fromHex("0xd0c621f5bb524fb68aa3631b4a0629bf6bc210fe30e237d9caf8bfb476686b82eb8e8460062d187d6e2699ddc8988c0c"), + ValidatorPubKey.fromHex("0x10eb53f3ba6d355e301c785a2f204294c6a63233edee9cc135791815d086c9a8604c0d46baca6abe8c7a58e708e2106a"), + ValidatorPubKey.fromHex("0x4244a5380986232f8fb39f9396be04e6c504c3b1f87e9672d7154d09b97f0fa86cae849aac06b30ce993e00e126cf5b0"), + ValidatorPubKey.fromHex("0x2382850a411c389df2afdd2a03a6196b451893e2674d11e0b8ac6914ffa53c7a1ced201cc1390a3aa1a2879dcdfa143b"), + ValidatorPubKey.fromHex("0xa20189e31ecc6a8c2002a9dec9645aada8f01dbaa6f22f7efcc10e1de109f2528edcbe768f1baf78b8ecba189d70e28b"), + ValidatorPubKey.fromHex("0xd1f4e4ebedcc39544148157f4a5279def61a8dda08c087afbcc85e85f5fe8a244972e26077cfc1820c0c85814adfad6e"), + ValidatorPubKey.fromHex("0xf62d8f1b982babdffcc6616f8b2ac54fac5224c7a1fb66121079b9a521aff4f2ade3cd7aa40baa838e522a927179ac82"), + ValidatorPubKey.fromHex("0x7e0c87bbf88d5762dfca69732bb36525d11a755fde736f28088bc17958cb8d5745a923a56c6c0b4e98c0ffd9623f9816"), + ValidatorPubKey.fromHex("0xbf1d6ae7fd84bee92a4e22bd73b3869402504736ff5af0da6e02181ae2506a248ca4e969a82ea0304a93b6bb68d29435"), + ValidatorPubKey.fromHex("0x8ec4826fcde422ba62d222274fda595cd988d27fa0ffcbc91ab7ace22d2c9617a94ba008064a5f159801dc3b1956d96f"), + ValidatorPubKey.fromHex("0x068bee5a0d17f286962fdf71fe6e9d8b2d05f8203ecf2fbc0672003ec18a53636062dabd430715b8599f1111091417dd"), + ValidatorPubKey.fromHex("0xc0e15eadc90fbf93e2deccdd58cb13b30fea11913ca39c2ee42ddf74201dae1e84553ce8c6818d91658cb8ae97573c24"), + ValidatorPubKey.fromHex("0x5a0e0446883b0a0f09ea42faffc02ebf25407159503f5b430a216a54b1b9a4272765314c267ee2f3be8fe101208a28fd"), + ValidatorPubKey.fromHex("0xc22aa9c85a08126c371c19163c940c459a478a7391cabfb170a352faa30687ef571568d4ad327a6fe69652cd0daa33af"), + ValidatorPubKey.fromHex("0xc53c961a6977d4711914b2852ac231e6dae019ce13555e189bcae94b1786f0bb3b3e8ad173c3f029758ecbc0c0b1c6f0"), + ValidatorPubKey.fromHex("0x925aefdfeaeea3402ddd678a7069c20183fed9a11f7f866215788177ba9ae9d2914874866c2dd78f79f81495ce172352"), + ValidatorPubKey.fromHex("0x4aca00821c817196db75be87cb044f36466c65e5ea3ca90c60353b3927107bdbd8ec0775dfe8c08ea123801f4443d01b"), + ValidatorPubKey.fromHex("0xb84960b4042210498cd2ab478685a1b65e2a4e3bbf2e813440e38f38659def0e5ebe9514316f125634e23ae398fa2458"), + ValidatorPubKey.fromHex("0x3dbee79b334a30be85c82ae64331ab0bd7ce371c2b5cc734212f079209a845d0f45393bbca97ffad203e0af81af4325b"), + ValidatorPubKey.fromHex("0xfd9e33dec3e8ebeeb2ec64297ace2997dc6ecf148d98067cc3aabf2419a2788160c4d670836419672eebd663999ba53b"), + ValidatorPubKey.fromHex("0xdd9de04d992ecd5991ed84567803f2195b9c0cbbf74968e60c2272ba59f741fb07e84eefd970a0507b36ad7e4bd56e7e"), + ValidatorPubKey.fromHex("0xdc8e45cb11fdf996418a930033d5fcb64ec69cf6ac4cf31bd371bd3fd576e370b98cd817caf18ad5f0074b24f7d27720"), + ValidatorPubKey.fromHex("0x341aabda461da1de4a70c254b4fb2a9d4a811a9b4c3f25559653d959d893636e1308fa18bfbfd6b4dc880ce989573425"), + ValidatorPubKey.fromHex("0xce846e2b8e9a1d7ea6e20c0645c710c5226b3fed662bb9e03f61d7d53c61fe3f1708737169f31b845905b553dccf98ea"), + ValidatorPubKey.fromHex("0xdd72c08651f9df0a7b1c765629fa498047c88013f767f46a2e5cf3fb1f677c3945c6fc68a4aa5c906acdbc07a7f5760d"), + ValidatorPubKey.fromHex("0x20bcc1ff7e47478d7427991ec79a4817a64359f83efa76ab9b5d6167abd989e984dc085546cbdec53e0b1250d808184e"), + ValidatorPubKey.fromHex("0xe16752498b4556447924ebfa1917c85e6c40591ee929051bfa3d73c64103c35823cc127b619ad418cdf574bf9e7bc8fc"), + ValidatorPubKey.fromHex("0x98f28110b52e2e86650bfd9410466c2ea331ed808eaa237e251c35aceef5df3b7ab22bdfd31adcceb3d4f53c6453936c"), + ValidatorPubKey.fromHex("0xd77b4fd7d53ebfd061ce285f872202892eb80fdef197474db4afccaf42ab3ce49ac9b5cea4762dd921d0ce82b0a94db4"), + ValidatorPubKey.fromHex("0xe3839842a9395a842c95f8ce12d252b1b9a9a530a63f211c5c0e86eed44f22dfccd255aab099d1831add9758e6edc630"), + ValidatorPubKey.fromHex("0x73ffbe5ed3bfc887ef2471c5503108d49d63ee01bbef1f1632a0d51ab9993bc0a2ef722f77a4fc5408b441a85c6d1319"), + ValidatorPubKey.fromHex("0x3fb9dc92802a74998079db011129cb47962f6b000992a378951dcd4a0d7e2c46ffe514495343f8b0f98fdcde454512e3"), + ValidatorPubKey.fromHex("0xff1bc41f9168a35c8e2cff51cb3cb2f0dcb81e98fd51fedc199127398736efe56438b840cd744c76529cda980de65962"), + ValidatorPubKey.fromHex("0x1abe1a28095abfd698c3a36a833a7441d4889022411319957eeab7ea3076d6fc6e0651b9f8913c92984e481da7efdd40"), + ValidatorPubKey.fromHex("0xc7953e7a922afa0881c729f087874ad765afebb2b6e4fc579186f7cb09a4760786cac8ffe969af4dbabd769516077b40"), + ValidatorPubKey.fromHex("0xa66b38b24acb204c04396cb960cb6669d41aa56f001ba36eb4ccdeb13d32bb896376312dc8c4827ab76581daf07cd389"), + ValidatorPubKey.fromHex("0x2414b7202ffdbf4a35464e67def794c4e1bbaf9c31049057b86208897a36bd658fd89639156213db67b31714a7ee1685"), + ValidatorPubKey.fromHex("0x1420677d08036c8d79d096b403768b5ebab97eff3faa52eba05d609382dfdfa71de966046cf8ee2013a440e916c17f5e"), + ValidatorPubKey.fromHex("0x23b555fafcec0d7415bce9d67583a6a5c21711715a64821c97d11f9878dbf87ee9a1dafcdf5bce3e5a883055e342a801"), + ValidatorPubKey.fromHex("0x37a3cef1dd8a764fe729c31038f316be87fa1c52ab1e4d25acfda5a57dac9fe58a4f6ed1fec50f7c486b5f266caa7424"), + ValidatorPubKey.fromHex("0x3dbfcf3210d3e9bcd4aff7051bb05f990eddaf04d95fd35b5fa357f90bcdf737171d1ed66a2e259eaa02fde016de6cde"), + ValidatorPubKey.fromHex("0x204aad5834d57cd257d8cf9da85af8962bb22a1afb374a1a2dd7c354311d6171bc27f64528306ffd5ed95c381a04dde1"), + ValidatorPubKey.fromHex("0x7d17a7d7a6c19d639da6037187371887ddbea526b0f8617e21ad393777d45058341c701c7500df80088690e83281b9ff"), + ValidatorPubKey.fromHex("0x6b268fe10926fd28de688ae1445fa76283896e12976545ee0e1529c330fa4b019b26b0a93ffb307e9f85806e1252cdd4"), + ValidatorPubKey.fromHex("0x31a4abc22802945fe326e2aab04d6ddc45c95d611350753eeff4ea43429b71dfe6bb12af8a0e89a9cd70ec135cdac8db"), + ValidatorPubKey.fromHex("0x2543a88b52f538004b486a00b58d3fc3bd4d6455633a1b5b3e2b838b1c43af0d8c657d1cb92c2fdab3dee471d427ff40"), + ValidatorPubKey.fromHex("0xa83ccefa4afbf8ceb2c73985ec6319f581f860920dcae1239af0a186e41399ac1918bc61d509e79169fa0c3ecb2bf727"), + ValidatorPubKey.fromHex("0x2c78cb837292eb207833a869c68538cf816846fb35d56e3af8b3b44bbd665018d4a0d070ad67dc569578b5ad07c5ab49"), + ValidatorPubKey.fromHex("0xb4c53e9c4ce3bfa7adf62a5c1de27d7d98ad0b68f5396603a6dea962e62484580b49ca3b93adbcae962045131992cabe"), + ValidatorPubKey.fromHex("0xa23c5aa88788a6b79c65df394da8816bd0c77843a3b3c45e0e25352e62f4fe617eee32ce6ccd776c5d9e3f9818d9dc04"), + ValidatorPubKey.fromHex("0xb675283ad1e4c818d97ab1c25eb8de03a28dd879a609e75e38abc3312abee8f817f4b375f3da1a5a8d26b0b78640ae93"), + ValidatorPubKey.fromHex("0xdad0b657aad9e2bbcdadaa2b21fda5e5abac21bcd7cc03629293d94e8653d388b9edf2bad61b5d5786af423483fc27a5"), + ValidatorPubKey.fromHex("0x024188d12bf492f5f7609ebb33a296a98ea174bbd85b2835297b939f0e3730901c46169b9fb9df45148cfc630ce07d7a"), + ValidatorPubKey.fromHex("0x75254539b896b7fefbdc9ccf3fde2a91c45a5e3882b2b53588568204a0a8bb97dd6fc8bb4f3bae67cf9393a48281601a"), + ValidatorPubKey.fromHex("0x4f647f752c4f63389e102a4b481a78fb7e16d536220c002f3a5c3f3aa3c72165370c0977fbcb58a2561ff5387974ca1b"), + ValidatorPubKey.fromHex("0xc67038bba68c2024268c7447639ffb4e96a737d8622878aa99141caedfe661a500740b94a4fb145e6440d633b900da4a"), + ValidatorPubKey.fromHex("0xbd6034a4dc88cde34713844b596f7738ddc373f30f4224cb665e854c75ebe2ad0f95a28db4b02efb82d9450c6c54b691"), + ValidatorPubKey.fromHex("0x982c60e8d94ba0d74f412d65fc8a567e2437ab274c664793d7a499f34bba9750678b995df5609d2dc9b5b1e35470ab89"), + ValidatorPubKey.fromHex("0xdf7065fc6c930e9e2ec2fc3a1ac774dfd95a275906ea43df5f7149776fbe866a3488443f9de3e59636af9a596efdda8c"), + ValidatorPubKey.fromHex("0x65cfdae721d55a7ea55026c39be299b79f28a825cae5356db8736088adde30cbe937fad2e942e27ec79a8eba887d0d97"), + ValidatorPubKey.fromHex("0xf7ca9f90e803f34565ca40c8b396f81fb4ec0a0f08e884c2ad34196122df59bd3a4cbe1612537b02f1e7a5fb191f853e"), + ValidatorPubKey.fromHex("0x3d81e3b6cbd7db0241e0626d64e00a8b0c8e64a66ec55a2872a4a322a73c684b9df612b8fc5b7949e7ee077c8de12886"), + ValidatorPubKey.fromHex("0xace1bbd8e0aba9c435184bdcce9a3663e792c3116acf1263f7f72be8f820a3fa89af62e0a042be940c0dfa62ab365c51"), + ValidatorPubKey.fromHex("0xed3d6d0cc64d402ab80e03b4de7b64bddfbf468434d2fa1501645f5e0b131451d0bb757c6cd99a74ef7f2c09a1844b88"), + ValidatorPubKey.fromHex("0x47b009c3c6d143e41ca9019b0cd9c9c80a9ad1265dd3733fd8a0844eefa358e72eed6186f28693f1c2e7d40dfd93daa2"), + ValidatorPubKey.fromHex("0x07fedce8f0d3c73e19a8e2d9a1e945756db7278e5f3bff1e8475628c02be32ad1374bb8826af1c05355e20d1d6f58a79"), + ValidatorPubKey.fromHex("0xb8e036e20e29eb8cb10efc07633a91080a00fb4e8981cc1a6c32f104ac544f51cd772f11865c80124b626831c70311db"), + ValidatorPubKey.fromHex("0x2152cf952943c4d978dbc5133d0f660d47291a3c7046c6c341b5361b415c9b260e34a60d72e95ac582c0ee173fbe298c"), + ValidatorPubKey.fromHex("0x2b2ec6c9303d0339c6ce9ad76fabe70b9172d2e3d7a65c25c8ca54c9532e2b243e4f434c8e4f6eaf31ee4ad579ef5047"), + ValidatorPubKey.fromHex("0xf97ac5f9e876dfead5c28faa5b81a03cd0d5d10db3255e07007c397a39ab5b3951fa5019c7d50511e32eeb217c022f97"), + ValidatorPubKey.fromHex("0x5df30c40e47b7e99b3d886a519c67a328bdeeb4bffbdb35b3b2fb1a255b2310ae6ec82fd4bb7cbef473c76c1b21fb6af"), + ValidatorPubKey.fromHex("0xedc08e12bf6dfcb5a488e2edb4c1ff3c3500aa3b86a8601d5e50d14b78b8f3a559728952a8dc4329d173d60244321014"), + ValidatorPubKey.fromHex("0x08e51326bcd56617f60192af020cd96c279f7861dddba86db7103517cc85598665f8a5c972573049684406c20d49ba9a"), + ValidatorPubKey.fromHex("0x67969f8cc333aa4df3b89ad7e3fb9ee9aa9338513e0a5854f86c0a53fdd0de0161ec61768d48bf6b09ce280f6bfbc2d8"), + ValidatorPubKey.fromHex("0x764ae0456941595a1723cb791a58ad5cc9f69171a2c88868d2be3c9711f2d69faa6fa18d2c676461c429d105ce0c438f"), + ValidatorPubKey.fromHex("0x0fc3c8805a22c1ac9e0279855cb9af40fb565f4ae634d8684cda925962b539d90b6695556af3c4ded7b0f59bc3164992"), + ValidatorPubKey.fromHex("0x56d97dfffdb4c0510712557668f16c71b5e23f863a85189488fbfd9fda22fcaabcf6a89c2038424baf333f77e3e08750"), + ValidatorPubKey.fromHex("0xf5402659851950b1d780be1adbc24bb00555cf958511efe449894278298ca5eaea06dc95aea24a7dfdb26d5e511a9c49"), + ValidatorPubKey.fromHex("0x93799b8575c4af7982b047996cf559a3caa6bed72c956506dfd8c52ce5a2b155ee65ff74a87c00e4023c39e684b52c50"), + ValidatorPubKey.fromHex("0x1e18ebd851e9bb41746c25249d9bf298235504172a3af7818c8645c78a63fbad7f51678774dd37c91ec7aa45ebf71b2c"), + ValidatorPubKey.fromHex("0x93b8f1f1f2dcc9241362ac8427180d0ad4289bdce419dc509a84659310e52424cbc5591cd97f8f1bf06d50222048eaae"), + ValidatorPubKey.fromHex("0xccbfc97454592b224136af85c87f553b118d88dcb67c9a77739c0715063e0c93cea1b323ff2d03eea6d807cb1fe91662"), + ValidatorPubKey.fromHex("0x05152fd70ac2916791bac13e4537141f5e48a4b3478e2434ec9a28bb7bd1f3322cd50e4b4ff71013e8a10690998e1615"), + ValidatorPubKey.fromHex("0xb586758d3a23c2568c327540620ac96227c0bae313ad867c066b2c8d44fe8eef40069fc27cb65dcc2b2716492d530c8a"), + ValidatorPubKey.fromHex("0x0e042382adde45efb4fcb15c246aadc72a6600456528db1206a012e35149a673fbc18144f025b4f48ba1d60832e954cc"), + ValidatorPubKey.fromHex("0x7e2d4015b9d19bfb8425f119f0e0de99cd18c63f6687d302d6129b690ab6d981c93b43b795af870444020fd21d44eeda"), + ValidatorPubKey.fromHex("0x0f21d5d7ab25cebc9e669eb7817bd58726811766551e4672b9c48f4401d18aacda6be15cee76bbb3428581bd958226b2"), + ValidatorPubKey.fromHex("0xa8a5610bbfa03b641646a9132aa6858cd2969f116cea3774f3429e82acdcdcd6276d5a8ddb629f2a43cadf1c11d47b86"), + ValidatorPubKey.fromHex("0xcf9a6236b081a03821714c6f3fe1e0c3cad6e364c70590a06bdb974a0989b0296a5aec31f776c448617f95f22bf35f37"), + ValidatorPubKey.fromHex("0x5fdf7cb380287d3b1e185d6444e67707c8c8138f26b4d21b0863b451f5024d9f6c36b560a7abcf9ebfebeb7857d22c91"), + ValidatorPubKey.fromHex("0x8f7ca4292265224d5cc9e2e5c1b2e28ac201cc2890ef0bb2a1d1448a99f1a9a0d871bd87523c15f81bc600b7adcd614f"), + ValidatorPubKey.fromHex("0x271f9d3c83947315f645a329fe31f930496d1325315e0b0303929db1fea279cbbb7abfbd63cdf1cf6a55d689377c3eb0"), + ValidatorPubKey.fromHex("0x3f062a6d6a9f29554680951605415013c20d6cecaa53c2a540520204bfbe30f4324652379da800f71cf515bdfd0c0356"), + ValidatorPubKey.fromHex("0xcc0247bd8e379d26b23ee118407e9c31f57d2c4b37b5fbeca93108702ef40030b74fcf1b1291f58792cc01e3d08d915a"), + ValidatorPubKey.fromHex("0x0a815516a56da73e476e715a9dbb1f97a24362fb9c9e304d18df5e25ad63e9a7074d343e4e7cc82a02a250c68cc1f7b8"), + ValidatorPubKey.fromHex("0xb4f606c8452095c153e5fa9b8d99a9e9ef150ebaa48de46941926f0d627b0f964ef5132bbd8a41c64433c62c9a839f1e"), + ValidatorPubKey.fromHex("0x48d2406f1c49ac30da158f8d5f9e93f3329b78edc2b6788f3c1d073cd4a5b5d5f28b4d9da3c8af6b431afc1ad1e6189a"), + ValidatorPubKey.fromHex("0x222bc67ec7b3a01e5435fe36fa3e65e1a780503c7dce5ad2016971f468eda413787cc7131d109e1025c23a23bc518897"), + ValidatorPubKey.fromHex("0x4def4bbc0a1724866755e37fbceb92a2182d9f2b646f0accca3e17e134c70e224d0e10f3280a4fdce36965bd8e97ce9f"), + ValidatorPubKey.fromHex("0xacc34baaf23f47ebb0bfb33bb8d3f6d7db33a2a43357b44814be37d5a05b1ed87de18628692eeed7f701802fc600eb76"), + ValidatorPubKey.fromHex("0xc15fd7a4df0b23726de242a7a98df1cdf0073be7f15d9f2b63cb76c20b19c961412f1bfb1b57660f2bc4316c745044d3"), + ValidatorPubKey.fromHex("0x213202d275fc352a3717809ad83d2845d375cdec756dc246c5624c01a99c64b35c89ebdd5c4e5680151e7a14558fece8"), + ValidatorPubKey.fromHex("0xee9804870785865e8b2fed70a18c4540839174b7b4372199c9aead159a5e33fb10ed2c55367c77c6ea19e46e6f68b76e"), + ValidatorPubKey.fromHex("0x85c220efc7552b4da736a1488f828948a00395490eef5d7455584cd4ff153daf2097d92f54f7afaacbac36ab0aa14770"), + ValidatorPubKey.fromHex("0x02f8e0af7f55aa635be0fb8542a96365baadeb96953b269adf93ebdeff121a292c3a760bc02b0dcc11a4fc7d0f5fd016"), + ValidatorPubKey.fromHex("0x68dcfdc8a0111ef6dd3d9e66a74e72e00c9ef74e84a29d446da4a52188142438c6448241d27aff2d5bd29f33dd5af53d"), + ValidatorPubKey.fromHex("0x11746eedf0dbd72974e1fe05760bb1c8f433b1356e89eb3ebff8b80998d5cb5a0039a30d66a96697efae6304d0c622d9"), + ValidatorPubKey.fromHex("0xf8d4089b72130d41f204d2a8041e8ca9253c097cc96a30944e24a9f2d367ec718d4812cba6aa9cb76ffe4e709faf279e"), + ValidatorPubKey.fromHex("0x1b268552b698c1b5f03df6a24ef167431ea50daa2d46091382acd7ee81a32c59f95be51699b7ff6b5005f1100fa2d79f"), + ValidatorPubKey.fromHex("0xf8a2120edc919bd33930eab1053636c661d9d5c76c431ea36b061b47863bd064fdf92ccb403f14b849f963540b907872"), + ValidatorPubKey.fromHex("0x62ed9fc80d29b0b718badf1b6a86bea6870ae6a42dc49a010aa540a54f2dbccdd5bfea6423e9f9364b06862066497676"), + ValidatorPubKey.fromHex("0x63b9989f1cdf949ef56b9bb930975a3d6de7478ecf5a7e13663596a2ac46487c13141c5f8a214159b7d9a6bd66482065"), + ValidatorPubKey.fromHex("0x5bec9e3cc45d40749bd1ec8bb94293c2100237741d53e2ea5563dbdb262effca4959be6938871d878a83a85108c5e5f4"), + ValidatorPubKey.fromHex("0xe016535d3988fedc2d5c0e970ed32fc260ef6552e4966b587f045f6b295c519a2944f757387c44838316ed5db1f18821"), + ValidatorPubKey.fromHex("0x58d2ce59fb8d93ab87c6a13c04261cdb29455528405770426dc56f21a0fc81d93b8071c9619dc06e978d7f54a7998d37"), + ValidatorPubKey.fromHex("0x0d8e02c02eced67122964d422c00cf94650261e456374534bd75776915abd0ab6ecb272175ff4721576533f81efcbef9"), + ValidatorPubKey.fromHex("0xbd1d1f66acf0cc16b259c03192fd23df1f0255904cbc1e57d45e2ddb23aa90e5e86e3763016c41d91a969913a84204df"), + ValidatorPubKey.fromHex("0xb44d6e213628d3c881a2bc70ffe347824c53d5f37adbadb955dfcdbfa5e7dc098a629ee0f1a8723b8f13530dc56479af"), + ValidatorPubKey.fromHex("0x6570a77a68ba5522b56ced31052f2f7f3c29ffae668cd8741c122b1b84913349c3d8d19dcff9e3f3576094f55273f047"), + ValidatorPubKey.fromHex("0x59f4ce46b322c54f4f416ad8d94a1dcbd04198036849337f88bc3c6f7d757f49a9b6cd18ca6f5b296fbef45ee5105aca"), + ValidatorPubKey.fromHex("0xdc8808fce77f5bb51990163bd58408a0864c502651e8291d74f3fb85044ff343e5eaa4b74ac7d15596e15938f894c2c3")] + +func findValidatorIndexBruteforce( + validators: openArray[Validator], bsv: BucketSortedValidators, + h2: ValidatorPubKey): Opt[ValidatorIndex] = + for validatorIndex in bsv.extraItems: + if validators[validatorIndex.distinctBase].pubkey == h2: + return Opt.some validatorIndex.ValidatorIndex + for validatorIndex in bsv.bucketSorted: + if validators[validatorIndex].pubkey == h2: + return Opt.some validatorIndex.ValidatorIndex + Opt.none ValidatorIndex + +suite "ValidatorPubKey bucket sort": + setup: + var hashedPubkeyItems = mapIt(pubkeys, HashedValidatorPubKeyItem( + key: it.get, root: hash_tree_root(it.get))) + let + hashedPubkeys = mapIt(hashedPubkeyItems, HashedValidatorPubKey( + value: unsafeAddr it)) + validators = mapIt(hashedPubkeys, Validator(pubkeyData: it)) + + test "one-shot construction": + let bsv = sortValidatorBuckets( + validators.toOpenArray(0, 7*validators.len div 8)) + for vidx in 0 ..< validators.len: + check findValidatorIndex(validators, bsv[], validators[vidx].pubkey) == + findValidatorIndexBruteforce(validators, bsv[], validators[vidx].pubkey) + + test "incremental construction": + let bsv = sortValidatorBuckets([]) + template rv: untyped = validators.toOpenArray(0, 7*validators.len div 8) + for vidx in 0 ..< len(rv): + bsv[].add vidx.ValidatorIndex + for vidx, validator in validators: + let foundIdx = findValidatorIndex(rv, bsv[], validator.pubkey) + check: + foundIdx == findValidatorIndexBruteforce(rv, bsv[], validator.pubkey) + foundIdx.isOk == (vidx < rv.len) \ No newline at end of file diff --git a/tests/teststateutil.nim b/tests/teststateutil.nim index 2eb8a12d1..bbcb79ad9 100644 --- a/tests/teststateutil.nim +++ b/tests/teststateutil.nim @@ -10,14 +10,12 @@ import chronicles, ./mocking/mock_deposits, - ../beacon_chain/spec/[ - forks, state_transition, state_transition_block] + ../beacon_chain/spec/[forks, state_transition] -from ".."/beacon_chain/bloomfilter import constructBloomFilter +from ".."/beacon_chain/validator_bucket_sort import sortValidatorBuckets from ".."/beacon_chain/spec/state_transition_epoch import get_validator_balance_after_epoch, process_epoch - func round_multiple_down(x: Gwei, n: Gwei): Gwei = ## Round the input to the previous multiple of "n" x - x mod n @@ -39,7 +37,7 @@ proc valid_deposit(state: var ForkyHashedBeaconState) = 0.Gwei doAssert process_deposit( defaultRuntimeConfig, state.data, - constructBloomFilter(state.data.validators.asSeq)[], deposit, {}).isOk + sortValidatorBuckets(state.data.validators.asSeq)[], deposit, {}).isOk doAssert state.data.validators.len == pre_val_count + 1 doAssert state.data.balances.len == pre_val_count + 1 doAssert state.data.balances.item(validator_index) == pre_balance + deposit.data.amount @@ -115,4 +113,4 @@ proc checkPerValidatorBalanceCalc*( i.ValidatorIndex): return false - true + true \ No newline at end of file