diff --git a/AllTests-mainnet.md b/AllTests-mainnet.md index 45848fdb3..57bf01b94 100644 --- a/AllTests-mainnet.md +++ b/AllTests-mainnet.md @@ -604,8 +604,10 @@ OK: 6/6 Fail: 0/6 Skip: 0/6 ```diff + Doppelganger for genesis validator OK + Doppelganger for validator that activates in same epoch as check OK ++ Dynamic validator set: queryValidatorsSource() test OK ++ Dynamic validator set: updateDynamicValidators() test OK ``` -OK: 2/2 Fail: 0/2 Skip: 0/2 +OK: 4/4 Fail: 0/4 Skip: 0/4 ## Zero signature sanity checks ```diff + SSZ serialization roundtrip of SignedBeaconBlockHeader OK @@ -700,4 +702,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 OK: 9/9 Fail: 0/9 Skip: 0/9 ---TOTAL--- -OK: 393/398 Fail: 0/398 Skip: 5/398 +OK: 395/400 Fail: 0/400 Skip: 5/400 diff --git a/beacon_chain/conf.nim b/beacon_chain/conf.nim index 4f8a32ef9..a82a51acc 100644 --- a/beacon_chain/conf.nim +++ b/beacon_chain/conf.nim @@ -37,7 +37,8 @@ export defaultEth2TcpPort, enabledLogLevel, ValidIpAddress, defs, parseCmdArg, completeCmdArg, network_metadata, el_conf, network, BlockHashOrNumber, - confTomlDefs, confTomlNet, confTomlUri + confTomlDefs, confTomlNet, confTomlUri, + LightClientDataImportMode declareGauge network_name, "network name", ["name"] @@ -165,6 +166,11 @@ type desc: "Remote Web3Signer URL that will be used as a source of validators" name: "validators-source"}: Option[string] + validatorsSourceInverval* {. + desc: "Number of minutes between validator list updates" + name: "validators-source-interval" + defaultValue: 60 .}: Natural + secretsDirFlag* {. desc: "A directory containing validator keystore passwords" name: "secrets-dir" .}: Option[InputDir] @@ -883,6 +889,11 @@ type desc: "Remote Web3Signer URL that will be used as a source of validators" name: "validators-source"}: Option[string] + validatorsSourceInverval* {. + desc: "Number of minutes between validator list updates" + name: "validators-source-interval" + defaultValue: 60 .}: Natural + secretsDirFlag* {. desc: "A directory containing validator keystore passwords" name: "secrets-dir" .}: Option[InputDir] diff --git a/beacon_chain/deposits.nim b/beacon_chain/deposits.nim index 703888c63..63d194894 100644 --- a/beacon_chain/deposits.nim +++ b/beacon_chain/deposits.nim @@ -8,7 +8,7 @@ import std/[os, sequtils, times], - stew/byteutils, + stew/[byteutils, base10], chronicles, ./spec/eth2_apis/rest_beacon_client, ./spec/signatures, @@ -220,15 +220,29 @@ proc restValidatorExit(config: BeaconNodeConf) {.async.} = quit 1 let signingFork = try: - let response = await client.getSpec() + let response = await client.getSpecVC() if response.status == 200: - let spec = response.data + let + spec = response.data.data + denebForkEpoch = + block: + let s = spec.getOrDefault("DENEB_FORK_EPOCH", $FAR_FUTURE_EPOCH) + Epoch(Base10.decode(uint64, s).get(uint64(FAR_FUTURE_EPOCH))) # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/beacon-chain.md#voluntary-exits # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/deneb/beacon-chain.md#modified-process_voluntary_exit - if currentEpoch >= Epoch(spec.data.DENEB_FORK_EPOCH): + if currentEpoch >= denebForkEpoch: + let capellaForkVersion = + block: + var res: Version + # CAPELLA_FOR_VERSION has specific format - "0x01000000", so + # default empty string is invalid, so `hexToByteArrayStrict` + # will raise exception on empty string. + let s = spec.getOrDefault("CAPELLA_FORK_VERSION", "") + hexToByteArrayStrict(s, distinctBase(res)) + res Fork( - current_version: spec.data.CAPELLA_FORK_VERSION, - previous_version: spec.data.CAPELLA_FORK_VERSION, + current_version: capellaForkVersion, + previous_version: capellaForkVersion, epoch: GENESIS_EPOCH) # irrelevant when current/previous identical else: fork @@ -239,6 +253,8 @@ proc restValidatorExit(config: BeaconNodeConf) {.async.} = reason = exc.msg quit 1 + debug "Signing fork obtained", fork = fork + if not config.printData: case askForExitConfirmation() of ClientExitAction.abort: @@ -249,7 +265,8 @@ proc restValidatorExit(config: BeaconNodeConf) {.async.} = var hadErrors = false for validator in validators: let restValidator = try: - let response = await client.getStateValidatorPlain(stateIdHead, validator.getIdent) + let response = await client.getStateValidatorPlain( + stateIdHead, validator.getIdent) if response.status == 200: let validatorInfo = decodeBytes(GetStateValidatorResponse, response.data, response.contentType) diff --git a/beacon_chain/gossip_processing/batch_validation.nim b/beacon_chain/gossip_processing/batch_validation.nim index 55648c8d4..1fa749754 100644 --- a/beacon_chain/gossip_processing/batch_validation.nim +++ b/beacon_chain/gossip_processing/batch_validation.nim @@ -228,6 +228,13 @@ proc batchVerifyTask(task: ptr BatchTask) {.nimcall.} = discard task[].signal.fireSync() +proc spawnBatchVerifyTask(tp: Taskpool, task: ptr BatchTask) = + # Inlining this `proc` leads to compilation problems on Nim 2.0 + # - Error: cannot generate destructor for generic type: Isolated + # Workaround: Ensure that `tp.spawn` is not used within an `{.async.}` proc + # Possibly related to: https://github.com/nim-lang/Nim/issues/22305 + tp.spawn batchVerifyTask(task) + proc batchVerifyAsync*( verifier: ref BatchVerifier, signal: ThreadSignalPtr, batch: ref Batch): Future[bool] {.async.} = @@ -245,7 +252,7 @@ proc batchVerifyAsync*( let taskPtr = addr task doAssert verifier[].taskpool.numThreads > 1, "Must have at least one separate thread or signal will never be fired" - verifier[].taskpool.spawn batchVerifyTask(taskPtr) + verifier[].taskpool.spawnBatchVerifyTask(taskPtr) await signal.wait() task.ok.load() diff --git a/beacon_chain/nimbus_beacon_node.nim b/beacon_chain/nimbus_beacon_node.nim index 0c160ded9..e2c77005c 100644 --- a/beacon_chain/nimbus_beacon_node.nim +++ b/beacon_chain/nimbus_beacon_node.nim @@ -1617,6 +1617,7 @@ proc run(node: BeaconNode) {.raises: [CatchableError].} = waitFor node.updateGossipStatus(wallSlot) + asyncSpawn pollForDynamicValidators(node) asyncSpawn runSlotLoop(node, wallTime, onSlotStart) asyncSpawn runOnSecondLoop(node) asyncSpawn runQueueProcessingLoop(node.blockProcessor) diff --git a/beacon_chain/nimbus_validator_client.nim b/beacon_chain/nimbus_validator_client.nim index 946fec1d6..fdd2b65be 100644 --- a/beacon_chain/nimbus_validator_client.nim +++ b/beacon_chain/nimbus_validator_client.nim @@ -90,10 +90,12 @@ proc initValidators(vc: ValidatorClientRef): Future[bool] {.async.} = var duplicates: seq[ValidatorPubKey] for keystore in listLoadableKeystores(vc.config, vc.keystoreCache): vc.addValidator(keystore) - let dynamicKeystores = await queryValidatorsSource(vc.config) - for keystore in dynamicKeystores: - vc.addValidator(keystore) - return true + let res = await queryValidatorsSource(vc.config) + if res.isOk(): + let dynamicKeystores = res.get() + for keystore in dynamicKeystores: + vc.addValidator(keystore) + true proc initClock(vc: ValidatorClientRef): Future[BeaconClock] {.async.} = # This procedure performs initialization of BeaconClock using current genesis diff --git a/beacon_chain/spec/eth2_apis/rest_config_calls.nim b/beacon_chain/spec/eth2_apis/rest_config_calls.nim index 2368e5d24..4987ba004 100644 --- a/beacon_chain/spec/eth2_apis/rest_config_calls.nim +++ b/beacon_chain/spec/eth2_apis/rest_config_calls.nim @@ -20,10 +20,6 @@ proc getForkSchedulePlain*(): RestPlainResponse {. rest, endpoint: "/eth/v1/config/fork_schedule", meth: MethodGet.} ## https://ethereum.github.io/beacon-APIs/#/Config/getForkSchedule -proc getSpec*(): RestResponse[GetSpecResponse] {. - rest, endpoint: "/eth/v1/config/spec", meth: MethodGet.} - ## https://ethereum.github.io/beacon-APIs/#/Config/getSpec - proc getSpecVC*(): RestResponse[GetSpecVCResponse] {. rest, endpoint: "/eth/v1/config/spec", meth: MethodGet.} ## https://ethereum.github.io/beacon-APIs/#/Config/getSpec diff --git a/beacon_chain/spec/eth2_apis/rest_types.nim b/beacon_chain/spec/eth2_apis/rest_types.nim index df75b0527..f5888330a 100644 --- a/beacon_chain/spec/eth2_apis/rest_types.nim +++ b/beacon_chain/spec/eth2_apis/rest_types.nim @@ -344,143 +344,6 @@ type of ConsensusFork.Capella: capellaData*: capella.BeaconBlock of ConsensusFork.Deneb: denebData*: DenebBlockContents - RestSpec* = object - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.1/presets/mainnet/phase0.yaml - MAX_COMMITTEES_PER_SLOT*: uint64 - TARGET_COMMITTEE_SIZE*: uint64 - MAX_VALIDATORS_PER_COMMITTEE*: uint64 - SHUFFLE_ROUND_COUNT*: uint64 - HYSTERESIS_QUOTIENT*: uint64 - HYSTERESIS_DOWNWARD_MULTIPLIER*: uint64 - HYSTERESIS_UPWARD_MULTIPLIER*: uint64 - MIN_DEPOSIT_AMOUNT*: uint64 - MAX_EFFECTIVE_BALANCE*: uint64 - EFFECTIVE_BALANCE_INCREMENT*: uint64 - MIN_ATTESTATION_INCLUSION_DELAY*: uint64 - SLOTS_PER_EPOCH*: uint64 - MIN_SEED_LOOKAHEAD*: uint64 - MAX_SEED_LOOKAHEAD*: uint64 - EPOCHS_PER_ETH1_VOTING_PERIOD*: uint64 - SLOTS_PER_HISTORICAL_ROOT*: uint64 - MIN_EPOCHS_TO_INACTIVITY_PENALTY*: uint64 - EPOCHS_PER_HISTORICAL_VECTOR*: uint64 - EPOCHS_PER_SLASHINGS_VECTOR*: uint64 - HISTORICAL_ROOTS_LIMIT*: uint64 - VALIDATOR_REGISTRY_LIMIT*: uint64 - BASE_REWARD_FACTOR*: uint64 - WHISTLEBLOWER_REWARD_QUOTIENT*: uint64 - PROPOSER_REWARD_QUOTIENT*: uint64 - INACTIVITY_PENALTY_QUOTIENT*: uint64 - MIN_SLASHING_PENALTY_QUOTIENT*: uint64 - PROPORTIONAL_SLASHING_MULTIPLIER*: uint64 - MAX_PROPOSER_SLASHINGS*: uint64 - MAX_ATTESTER_SLASHINGS*: uint64 - MAX_ATTESTATIONS*: uint64 - MAX_DEPOSITS*: uint64 - MAX_VOLUNTARY_EXITS*: uint64 - - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/presets/mainnet/altair.yaml - INACTIVITY_PENALTY_QUOTIENT_ALTAIR*: uint64 - MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR*: uint64 - PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR*: uint64 - SYNC_COMMITTEE_SIZE*: uint64 - EPOCHS_PER_SYNC_COMMITTEE_PERIOD*: uint64 - MIN_SYNC_COMMITTEE_PARTICIPANTS*: uint64 - UPDATE_TIMEOUT*: uint64 - - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/presets/mainnet/bellatrix.yaml - INACTIVITY_PENALTY_QUOTIENT_BELLATRIX*: uint64 - MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX*: uint64 - PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX*: uint64 - MAX_BYTES_PER_TRANSACTION*: uint64 - MAX_TRANSACTIONS_PER_PAYLOAD*: uint64 - BYTES_PER_LOGS_BLOOM*: uint64 - MAX_EXTRA_DATA_BYTES*: uint64 - - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/presets/mainnet/capella.yaml - MAX_BLS_TO_EXECUTION_CHANGES*: uint64 - MAX_WITHDRAWALS_PER_PAYLOAD*: uint64 - MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP*: uint64 - - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/configs/mainnet.yaml - PRESET_BASE*: string - CONFIG_NAME*: string - TERMINAL_TOTAL_DIFFICULTY*: UInt256 - TERMINAL_BLOCK_HASH*: BlockHash - TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH*: uint64 - MIN_GENESIS_ACTIVE_VALIDATOR_COUNT*: uint64 - MIN_GENESIS_TIME*: uint64 - GENESIS_FORK_VERSION*: Version - GENESIS_DELAY*: uint64 - ALTAIR_FORK_VERSION*: Version - ALTAIR_FORK_EPOCH*: uint64 - BELLATRIX_FORK_VERSION*: Version - BELLATRIX_FORK_EPOCH*: uint64 - CAPELLA_FORK_VERSION*: Version - CAPELLA_FORK_EPOCH*: uint64 - DENEB_FORK_VERSION*: Version - DENEB_FORK_EPOCH*: uint64 - SECONDS_PER_SLOT*: uint64 - SECONDS_PER_ETH1_BLOCK*: uint64 - MIN_VALIDATOR_WITHDRAWABILITY_DELAY*: uint64 - SHARD_COMMITTEE_PERIOD*: uint64 - ETH1_FOLLOW_DISTANCE*: uint64 - INACTIVITY_SCORE_BIAS*: uint64 - INACTIVITY_SCORE_RECOVERY_RATE*: uint64 - EJECTION_BALANCE*: uint64 - MIN_PER_EPOCH_CHURN_LIMIT*: uint64 - CHURN_LIMIT_QUOTIENT*: uint64 - PROPOSER_SCORE_BOOST*: uint64 - DEPOSIT_CHAIN_ID*: uint64 - DEPOSIT_NETWORK_ID*: uint64 - DEPOSIT_CONTRACT_ADDRESS*: Eth1Address - - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/beacon-chain.md#constants - # GENESIS_SLOT - # GENESIS_EPOCH - # FAR_FUTURE_EPOCH - # BASE_REWARDS_PER_EPOCH - # DEPOSIT_CONTRACT_TREE_DEPTH - # JUSTIFICATION_BITS_LENGTH - # ENDIANNESS - BLS_WITHDRAWAL_PREFIX*: RestWithdrawalPrefix - ETH1_ADDRESS_WITHDRAWAL_PREFIX*: RestWithdrawalPrefix - DOMAIN_BEACON_PROPOSER*: DomainType - DOMAIN_BEACON_ATTESTER*: DomainType - DOMAIN_RANDAO*: DomainType - DOMAIN_DEPOSIT*: DomainType - DOMAIN_VOLUNTARY_EXIT*: DomainType - DOMAIN_SELECTION_PROOF*: DomainType - DOMAIN_AGGREGATE_AND_PROOF*: DomainType - - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/altair/beacon-chain.md#constants - TIMELY_SOURCE_FLAG_INDEX*: byte - TIMELY_TARGET_FLAG_INDEX*: byte - TIMELY_HEAD_FLAG_INDEX*: byte - TIMELY_SOURCE_WEIGHT*: uint64 - TIMELY_TARGET_WEIGHT*: uint64 - TIMELY_HEAD_WEIGHT*: uint64 - SYNC_REWARD_WEIGHT*: uint64 - PROPOSER_WEIGHT*: uint64 - WEIGHT_DENOMINATOR*: uint64 - DOMAIN_SYNC_COMMITTEE*: DomainType - DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF*: DomainType - DOMAIN_CONTRIBUTION_AND_PROOF*: DomainType - # PARTICIPATION_FLAG_WEIGHTS - - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/capella/beacon-chain.md#domain-types - DOMAIN_BLS_TO_EXECUTION_CHANGE*: DomainType - - # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#constants - TARGET_AGGREGATORS_PER_COMMITTEE*: uint64 - RANDOM_SUBNETS_PER_VALIDATOR*: uint64 - EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION*: uint64 - ATTESTATION_SUBNET_COUNT*: uint64 - - # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/validator.md#constants - TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE*: uint64 - SYNC_COMMITTEE_SUBNET_COUNT*: uint64 - VCRuntimeConfig* = Table[string, string] RestDepositContract* = object @@ -665,7 +528,6 @@ type GetPoolProposerSlashingsResponse* = DataEnclosedObject[seq[ProposerSlashing]] GetPoolVoluntaryExitsResponse* = DataEnclosedObject[seq[SignedVoluntaryExit]] GetProposerDutiesResponse* = DataRootEnclosedObject[seq[RestProposerDuty]] - GetSpecResponse* = DataEnclosedObject[RestSpec] GetSpecVCResponse* = DataEnclosedObject[VCRuntimeConfig] GetStateFinalityCheckpointsResponse* = DataEnclosedObject[RestBeaconStatesFinalityCheckpoints] GetStateForkResponse* = DataEnclosedObject[Fork] diff --git a/beacon_chain/spec/forks.nim b/beacon_chain/spec/forks.nim index 9953964a2..a27e29c97 100644 --- a/beacon_chain/spec/forks.nim +++ b/beacon_chain/spec/forks.nim @@ -254,6 +254,31 @@ type capella*: ForkDigest deneb*: ForkDigest +template kind*( + x: typedesc[ + phase0.HashedBeaconState]): ConsensusFork = + ConsensusFork.Phase0 + +template kind*( + x: typedesc[ + altair.HashedBeaconState]): ConsensusFork = + ConsensusFork.Altair + +template kind*( + x: typedesc[ + bellatrix.HashedBeaconState]): ConsensusFork = + ConsensusFork.Bellatrix + +template kind*( + x: typedesc[ + capella.HashedBeaconState]): ConsensusFork = + ConsensusFork.Capella + +template kind*( + x: typedesc[ + deneb.HashedBeaconState]): ConsensusFork = + ConsensusFork.Deneb + macro getSymbolFromForkModule(fork: static ConsensusFork, symbolName: static string): untyped = let moduleName = case fork diff --git a/beacon_chain/spec/state_transition.nim b/beacon_chain/spec/state_transition.nim index 00598ba9e..f98731daa 100644 --- a/beacon_chain/spec/state_transition.nim +++ b/beacon_chain/spec/state_transition.nim @@ -335,10 +335,9 @@ proc state_transition*( state_transition_block( cfg, state, signedBlock, cache, flags, rollback) -# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#preparing-for-a-beaconblock template partialBeaconBlock*( cfg: RuntimeConfig, - state: var phase0.HashedBeaconState, + state: var ForkyHashedBeaconState, proposer_index: ValidatorIndex, randao_reveal: ValidatorSig, eth1_data: Eth1Data, @@ -347,13 +346,16 @@ template partialBeaconBlock*( deposits: seq[Deposit], validator_changes: BeaconBlockValidatorChanges, sync_aggregate: SyncAggregate, - execution_payload: bellatrix.ExecutionPayloadForSigning): - phase0.BeaconBlock = - phase0.BeaconBlock( + execution_payload: ForkyExecutionPayloadForSigning +): auto = + const consensusFork = typeof(state).kind + + # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#preparing-for-a-beaconblock + var res = consensusFork.BeaconBlockType( slot: state.data.slot, proposer_index: proposer_index.uint64, parent_root: state.latest_block_root, - body: phase0.BeaconBlockBody( + body: consensusFork.BeaconBlockBodyType( randao_reveal: randao_reveal, eth1_data: eth1data, graffiti: graffiti, @@ -363,127 +365,24 @@ template partialBeaconBlock*( deposits: List[Deposit, Limit MAX_DEPOSITS](deposits), voluntary_exits: validator_changes.voluntary_exits)) -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/validator.md#preparing-a-beaconblock -template partialBeaconBlock*( - cfg: RuntimeConfig, - state: var altair.HashedBeaconState, - proposer_index: ValidatorIndex, - randao_reveal: ValidatorSig, - eth1_data: Eth1Data, - graffiti: GraffitiBytes, - attestations: seq[Attestation], - deposits: seq[Deposit], - validator_changes: BeaconBlockValidatorChanges, - sync_aggregate: SyncAggregate, - execution_payload: bellatrix.ExecutionPayloadForSigning): - altair.BeaconBlock = - altair.BeaconBlock( - slot: state.data.slot, - proposer_index: proposer_index.uint64, - parent_root: state.latest_block_root, - body: altair.BeaconBlockBody( - randao_reveal: randao_reveal, - eth1_data: eth1data, - graffiti: graffiti, - proposer_slashings: validator_changes.proposer_slashings, - attester_slashings: validator_changes.attester_slashings, - attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations), - deposits: List[Deposit, Limit MAX_DEPOSITS](deposits), - voluntary_exits: validator_changes.voluntary_exits, - sync_aggregate: sync_aggregate)) + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/validator.md#preparing-a-beaconblock + when consensusFork >= ConsensusFork.Altair: + res.body.sync_aggregate = sync_aggregate -# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/bellatrix/validator.md#block-proposal -template partialBeaconBlock*( - cfg: RuntimeConfig, - state: var bellatrix.HashedBeaconState, - proposer_index: ValidatorIndex, - randao_reveal: ValidatorSig, - eth1_data: Eth1Data, - graffiti: GraffitiBytes, - attestations: seq[Attestation], - deposits: seq[Deposit], - validator_changes: BeaconBlockValidatorChanges, - sync_aggregate: SyncAggregate, - execution_payload: bellatrix.ExecutionPayloadForSigning): - bellatrix.BeaconBlock = - bellatrix.BeaconBlock( - slot: state.data.slot, - proposer_index: proposer_index.uint64, - parent_root: state.latest_block_root, - body: bellatrix.BeaconBlockBody( - randao_reveal: randao_reveal, - eth1_data: eth1data, - graffiti: graffiti, - proposer_slashings: validator_changes.proposer_slashings, - attester_slashings: validator_changes.attester_slashings, - attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations), - deposits: List[Deposit, Limit MAX_DEPOSITS](deposits), - voluntary_exits: validator_changes.voluntary_exits, - sync_aggregate: sync_aggregate, - execution_payload: execution_payload.executionPayload)) + # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/bellatrix/validator.md#block-proposal + when consensusFork >= ConsensusFork.Bellatrix: + res.body.execution_payload = execution_payload.executionPayload -# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/capella/validator.md#block-proposal -template partialBeaconBlock*( - cfg: RuntimeConfig, - state: var capella.HashedBeaconState, - proposer_index: ValidatorIndex, - randao_reveal: ValidatorSig, - eth1_data: Eth1Data, - graffiti: GraffitiBytes, - attestations: seq[Attestation], - deposits: seq[Deposit], - validator_changes: BeaconBlockValidatorChanges, - sync_aggregate: SyncAggregate, - execution_payload: capella.ExecutionPayloadForSigning): - capella.BeaconBlock = - capella.BeaconBlock( - slot: state.data.slot, - proposer_index: proposer_index.uint64, - parent_root: state.latest_block_root, - body: capella.BeaconBlockBody( - randao_reveal: randao_reveal, - eth1_data: eth1data, - graffiti: graffiti, - proposer_slashings: validator_changes.proposer_slashings, - attester_slashings: validator_changes.attester_slashings, - attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations), - deposits: List[Deposit, Limit MAX_DEPOSITS](deposits), - voluntary_exits: validator_changes.voluntary_exits, - sync_aggregate: sync_aggregate, - execution_payload: execution_payload.executionPayload, - bls_to_execution_changes: validator_changes.bls_to_execution_changes)) + # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/capella/validator.md#block-proposal + when consensusFork >= ConsensusFork.Capella: + res.body.bls_to_execution_changes = + validator_changes.bls_to_execution_changes -# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/validator.md#constructing-the-beaconblockbody -template partialBeaconBlock*( - cfg: RuntimeConfig, - state: var deneb.HashedBeaconState, - proposer_index: ValidatorIndex, - randao_reveal: ValidatorSig, - eth1_data: Eth1Data, - graffiti: GraffitiBytes, - attestations: seq[Attestation], - deposits: seq[Deposit], - validator_changes: BeaconBlockValidatorChanges, - sync_aggregate: SyncAggregate, - execution_payload: deneb.ExecutionPayloadForSigning): - deneb.BeaconBlock = - deneb.BeaconBlock( - slot: state.data.slot, - proposer_index: proposer_index.uint64, - parent_root: state.latest_block_root, - body: deneb.BeaconBlockBody( - randao_reveal: randao_reveal, - eth1_data: eth1data, - graffiti: graffiti, - proposer_slashings: validator_changes.proposer_slashings, - attester_slashings: validator_changes.attester_slashings, - attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations), - deposits: List[Deposit, Limit MAX_DEPOSITS](deposits), - voluntary_exits: validator_changes.voluntary_exits, - sync_aggregate: sync_aggregate, - execution_payload: execution_payload.executionPayload, - bls_to_execution_changes: validator_changes.bls_to_execution_changes, - blob_kzg_commitments: execution_payload.kzgs)) + # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/validator.md#constructing-the-beaconblockbody + when consensusFork >= ConsensusFork.Deneb: + res.body.blob_kzg_commitments = execution_payload.kzgs + + res proc makeBeaconBlock*( cfg: RuntimeConfig, diff --git a/beacon_chain/statediff.nim b/beacon_chain/statediff.nim index 456de12f7..dc01d9fcd 100644 --- a/beacon_chain/statediff.nim +++ b/beacon_chain/statediff.nim @@ -11,19 +11,19 @@ import stew/assign2, ./spec/forks -func diffModIncEpoch[T, U](hl: HashArray[U, T], startSlot: uint64): +func diffModIncEpoch[maxLen, T](hl: HashArray[maxLen, T], startSlot: uint64): array[SLOTS_PER_EPOCH, T] = - static: doAssert U.uint64 mod SLOTS_PER_EPOCH == 0 + static: doAssert maxLen.uint64 mod SLOTS_PER_EPOCH == 0 doAssert startSlot mod SLOTS_PER_EPOCH == 0 for i in startSlot ..< startSlot + SLOTS_PER_EPOCH: - result[i mod SLOTS_PER_EPOCH] = hl[i mod U.uint64] + result[i mod SLOTS_PER_EPOCH] = hl[i mod maxLen.uint64] -func applyModIncrement[T, U]( - ha: var HashArray[U, T], hl: array[SLOTS_PER_EPOCH, T], slot: uint64) = +func applyModIncrement[maxLen, T]( + ha: var HashArray[maxLen, T], hl: array[SLOTS_PER_EPOCH, T], slot: uint64) = var indexSlot = slot for item in hl: - ha[indexSlot mod U.uint64] = item + ha[indexSlot mod maxLen.uint64] = item indexSlot += 1 func applyValidatorIdentities( @@ -51,9 +51,9 @@ func setValidatorStatusesNoWithdrawals( validator[].exit_epoch = hl[i].exit_epoch validator[].withdrawable_epoch = hl[i].withdrawable_epoch -func replaceOrAddEncodeEth1Votes[T, U]( - votes0: openArray[T], votes0_len: int, votes1: HashList[T, U]): - (bool, List[T, U]) = +func replaceOrAddEncodeEth1Votes[T, maxLen]( + votes0: openArray[T], votes0_len: int, votes1: HashList[T, maxLen]): + (bool, List[T, maxLen]) = let num_votes0 = votes0.len lower_bound = @@ -67,17 +67,17 @@ func replaceOrAddEncodeEth1Votes[T, U]( else: num_votes0 - var res = (lower_bound == 0, default(List[T, U])) + var res = (lower_bound == 0, default(List[T, maxLen])) for i in lower_bound ..< votes1.len: if not result[1].add votes1[i]: raiseAssert "same limit" res -func replaceOrAddDecodeEth1Votes[T, U]( - votes0: var HashList[T, U], eth1_data_votes_replaced: bool, - votes1: List[T, U]) = +func replaceOrAddDecodeEth1Votes[T, maxLen]( + votes0: var HashList[T, maxLen], eth1_data_votes_replaced: bool, + votes1: List[T, maxLen]) = if eth1_data_votes_replaced: - votes0 = HashList[T, U]() + votes0 = HashList[T, maxLen]() for item in votes1: if not votes0.add item: @@ -209,7 +209,8 @@ func applyDiff*( state: var capella.BeaconState, immutableValidators: openArray[ImmutableValidatorData2], stateDiff: BeaconStateDiff) = - template assign[T, U](tgt: var HashList[T, U], src: List[T, U]) = + template assign[T, maxLen]( + tgt: var HashList[T, maxLen], src: List[T, maxLen]) = assign(tgt.data, src) tgt.resetCache() diff --git a/beacon_chain/validator_client/duties_service.nim b/beacon_chain/validator_client/duties_service.nim index 00e621d16..eb69c249d 100644 --- a/beacon_chain/validator_client/duties_service.nim +++ b/beacon_chain/validator_client/duties_service.nim @@ -19,7 +19,7 @@ logScope: service = ServiceName type DutiesServiceLoop* = enum AttesterLoop, ProposerLoop, IndicesLoop, SyncCommitteeLoop, - ProposerPreparationLoop, ValidatorRegisterLoop + ProposerPreparationLoop, ValidatorRegisterLoop, DynamicValidatorsLoop chronicles.formatIt(DutiesServiceLoop): case it @@ -29,6 +29,7 @@ chronicles.formatIt(DutiesServiceLoop): of SyncCommitteeLoop: "sync_committee_loop" of ProposerPreparationLoop: "proposer_prepare_loop" of ValidatorRegisterLoop: "validator_register_loop" + of DynamicValidatorsLoop: "dynamic_validators_loop" proc checkDuty(duty: RestAttesterDuty): bool = (duty.committee_length <= MAX_VALIDATORS_PER_COMMITTEE) and @@ -588,6 +589,38 @@ proc validatorIndexLoop(service: DutiesServiceRef) {.async.} = await service.pollForValidatorIndices() await service.waitForNextSlot() +proc dynamicValidatorsLoop*(service: DutiesServiceRef) {.async.} = + let vc = service.client + doAssert(vc.config.validatorsSourceInverval > 0) + + proc addValidatorProc(data: KeystoreData) = + vc.addValidator(data) + + var + timeout = minutes(vc.config.validatorsSourceInverval) + exitLoop = false + + while not(exitLoop): + exitLoop = + try: + await sleepAsync(timeout) + timeout = + block: + let res = await vc.config.queryValidatorsSource() + if res.isOk(): + let keystores = res.get() + debug "Validators source has been polled for validators", + keystores_found = len(keystores), + validators_source = vc.config.validatorsSource + vc.attachedValidators.updateDynamicValidators(keystores, + addValidatorProc) + minutes(vc.config.validatorsSourceInverval) + else: + seconds(5) + false + except CancelledError: + true + proc proposerPreparationsLoop(service: DutiesServiceRef) {.async.} = let vc = service.client debug "Beacon proposer preparation loop is waiting for initialization" @@ -661,6 +694,12 @@ proc mainLoop(service: DutiesServiceRef) {.async.} = service.validatorRegisterLoop() else: nil + dynamicFut = + if vc.config.validatorsSourceInverval > 0: + service.dynamicValidatorsLoop() + else: + debug "Dynamic validators update loop disabled" + nil while true: # This loop could look much more nicer/better, when @@ -673,7 +712,8 @@ proc mainLoop(service: DutiesServiceRef) {.async.} = FutureBase(proposeFut), FutureBase(indicesFut), FutureBase(syncFut), - FutureBase(prepareFut) + FutureBase(prepareFut), + FutureBase(dynamicFut) ] if not(isNil(registerFut)): futures.add(FutureBase(registerFut)) discard await race(futures) @@ -687,6 +727,9 @@ proc mainLoop(service: DutiesServiceRef) {.async.} = if not(isNil(registerFut)): checkAndRestart(ValidatorRegisterLoop, registerFut, service.validatorRegisterLoop()) + if not(isNil(dynamicFut)): + checkAndRestart(DynamicValidatorsLoop, dynamicFut, + service.dynamicValidatorsLoop()) false except CancelledError: debug "Service interrupted" @@ -703,6 +746,8 @@ proc mainLoop(service: DutiesServiceRef) {.async.} = pending.add(prepareFut.cancelAndWait()) if not(isNil(registerFut)) and not(registerFut.finished()): pending.add(registerFut.cancelAndWait()) + if not(isNil(dynamicFut)) and not(dynamicFut.finished()): + pending.add(dynamicFut.cancelAndWait()) if not(isNil(service.pollingAttesterDutiesTask)) and not(service.pollingAttesterDutiesTask.finished()): pending.add(service.pollingAttesterDutiesTask.cancelAndWait()) diff --git a/beacon_chain/validators/beacon_validators.nim b/beacon_chain/validators/beacon_validators.nim index a855cf82d..54c136abe 100644 --- a/beacon_chain/validators/beacon_validators.nim +++ b/beacon_chain/validators/beacon_validators.nim @@ -138,7 +138,12 @@ proc addValidators*(node: BeaconNode) = let dynamicStores = try: - waitFor(queryValidatorsSource(node.config)) + let res = waitFor(queryValidatorsSource(node.config)) + if res.isErr(): + # Error is already reported via log warning. + default(seq[KeystoreData]) + else: + res.get() except CatchableError as exc: warn "Unexpected error happens while polling validator's source", error = $exc.name, reason = $exc.msg @@ -161,6 +166,48 @@ proc addValidators*(node: BeaconNode) = gasLimit) v.updateValidator(data) +proc pollForDynamicValidators*(node: BeaconNode) {.async.} = + if node.config.validatorsSourceInverval == 0: + return + + proc addValidatorProc(keystore: KeystoreData) = + let + epoch = node.currentSlot().epoch + index = Opt.none(ValidatorIndex) + feeRecipient = + node.consensusManager[].getFeeRecipient(keystore.pubkey, index, epoch) + gasLimit = + node.consensusManager[].getGasLimit(keystore.pubkey) + discard node.attachedValidators[].addValidator(keystore, feeRecipient, + gasLimit) + + var + timeout = minutes(node.config.validatorsSourceInverval) + exitLoop = false + + while not(exitLoop): + exitLoop = + try: + await sleepAsync(timeout) + timeout = + block: + let res = await node.config.queryValidatorsSource() + if res.isOk(): + let keystores = res.get() + debug "Validators source has been polled for validators", + keystores_found = len(keystores), + validators_source = node.config.validatorsSource + node.attachedValidators.updateDynamicValidators(keystores, + addValidatorProc) + minutes(node.config.validatorsSourceInverval) + else: + # In case of error we going to repeat our call with much smaller + # interval. + seconds(5) + false + except CancelledError: + true + proc getValidator*(node: BeaconNode, idx: ValidatorIndex): Opt[AttachedValidator] = let key = ? node.dag.validatorKey(idx) node.attachedValidators[].getValidator(key.toPubKey()) @@ -435,11 +482,6 @@ proc makeBeaconBlockForHeadAndSlot*( exits = withState(state[]): node.validatorChangePool[].getBeaconBlockValidatorChanges( node.dag.cfg, forkyState.data) - syncAggregate = - if slot.epoch >= node.dag.cfg.ALTAIR_FORK_EPOCH: - node.syncCommitteeMsgPool[].produceSyncAggregate(head.bid, slot) - else: - SyncAggregate.init() payload = (await payloadFut).valueOr: beacon_block_production_errors.inc() warn "Unable to get execution payload. Skipping block proposal", @@ -456,7 +498,7 @@ proc makeBeaconBlockForHeadAndSlot*( attestations, eth1Proposal.deposits, exits, - syncAggregate, + node.syncCommitteeMsgPool[].produceSyncAggregate(head.bid, slot), payload, noRollback, # Temporary state - no need for rollback cache, @@ -1492,9 +1534,7 @@ proc getValidatorRegistration( proc registerValidators*(node: BeaconNode, epoch: Epoch) {.async.} = try: - if (not node.config.payloadBuilderEnable) or - node.currentSlot.epoch < node.dag.cfg.BELLATRIX_FORK_EPOCH: - return + if not node.config.payloadBuilderEnable: return const HttpOk = 200 diff --git a/beacon_chain/validators/keystore_management.nim b/beacon_chain/validators/keystore_management.nim index b91b650e4..177fbe32a 100644 --- a/beacon_chain/validators/keystore_management.nim +++ b/beacon_chain/validators/keystore_management.nim @@ -95,6 +95,8 @@ type MultipleKeystoresDecryptor* = object previouslyUsedPassword*: string + QueryResult = Result[seq[KeystoreData], string] + const minPasswordLen = 12 minPasswordEntropy = 60.0 @@ -627,12 +629,11 @@ proc existsKeystore(keystoreDir: string, return true false -proc queryValidatorsSource*(config: AnyConf): Future[seq[KeystoreData]] {. - async.} = +proc queryValidatorsSource*(config: AnyConf): Future[QueryResult] {.async.} = var keystores: seq[KeystoreData] if config.validatorsSource.isNone() or len(config.validatorsSource.get()) == 0: - return keystores + return QueryResult.ok(keystores) let vsource = config.validatorsSource.get() logScope: @@ -647,9 +648,9 @@ proc queryValidatorsSource*(config: AnyConf): Future[seq[KeystoreData]] {. let res = RestClientRef.new(vsource, prestoFlags, httpFlags, socketFlags = socketFlags) if res.isErr(): - # TODO keep trying in case of temporary network failure - warn "Unable to resolve validator's source distributed signer address" - return keystores + warn "Unable to resolve validator's source distributed signer " & + "address", reason = $res.error + return QueryResult.err($res.error) res.get() keys = try: @@ -657,26 +658,27 @@ proc queryValidatorsSource*(config: AnyConf): Future[seq[KeystoreData]] {. if response.status != 200: warn "Remote validator's source responded with error", error = response.status - return keystores + return QueryResult.err( + "Remote validator's source responded with error [" & + $response.status & "]") let res = decodeBytes(Web3SignerKeysResponse, response.data, response.contentType) if res.isErr(): warn "Unable to obtain validator's source response", reason = res.error - return keystores - + return QueryResult.err($res.error) res.get() except RestError as exc: warn "Unable to poll validator's source", reason = $exc.msg - return keystores + return QueryResult.err($exc.msg) except CancelledError as exc: debug "The polling of validator's source was interrupted" raise exc except CatchableError as exc: warn "Unexpected error occured while polling validator's source", error = $exc.name, reason = $exc.msg - return keystores + return QueryResult.err($exc.msg) for pubkey in keys: keystores.add(KeystoreData( @@ -689,7 +691,7 @@ proc queryValidatorsSource*(config: AnyConf): Future[seq[KeystoreData]] {. flags: {RemoteKeystoreFlag.DynamicKeystore}, remoteType: RemoteSignerType.Web3Signer)) - keystores + QueryResult.ok(keystores) iterator listLoadableKeys*(validatorsDir, secretsDir: string, keysMask: set[KeystoreKind]): CookedPubKey = diff --git a/beacon_chain/validators/validator_pool.nim b/beacon_chain/validators/validator_pool.nim index 8f6080082..1808f907c 100644 --- a/beacon_chain/validators/validator_pool.nim +++ b/beacon_chain/validators/validator_pool.nim @@ -87,6 +87,8 @@ type slashingProtection*: SlashingProtectionDB doppelgangerDetectionEnabled*: bool + AddValidatorProc* = proc(keystore: KeystoreData) {.gcsafe, raises: [].} + template pubkey*(v: AttachedValidator): ValidatorPubKey = v.data.pubkey @@ -221,8 +223,12 @@ proc removeValidator*(pool: var ValidatorPool, pubkey: ValidatorPubKey) = of ValidatorKind.Local: notice "Local validator detached", pubkey, validator = shortLog(validator) of ValidatorKind.Remote: - notice "Remote validator detached", pubkey, - validator = shortLog(validator) + if RemoteKeystoreFlag.DynamicKeystore in validator.data.flags: + notice "Dynamic remote validator detached", pubkey, + validator = shortLog(validator) + else: + notice "Remote validator detached", pubkey, + validator = shortLog(validator) validators.set(pool.count().int64) func needsUpdate*(validator: AttachedValidator): bool = @@ -373,6 +379,46 @@ func triggersDoppelganger*( let v = pool.getValidator(pubkey) v.isSome() and v[].triggersDoppelganger(epoch) +proc updateDynamicValidators*(pool: ref ValidatorPool, + keystores: openArray[KeystoreData], + addProc: AddValidatorProc) = + var + keystoresTable: Table[ValidatorPubKey, Opt[KeystoreData]] + deleteValidators: seq[ValidatorPubKey] + + for keystore in keystores: + keystoresTable[keystore.pubkey] = Opt.some(keystore) + + # We preserve `Local` and `Remote` keystores which are not from dynamic set, + # and also we removing all the dynamic keystores which are not part of new + # dynamic set. + for validator in pool[].items(): + if validator.kind == ValidatorKind.Remote: + if RemoteKeystoreFlag.DynamicKeystore in validator.data.flags: + let keystore = keystoresTable.getOrDefault(validator.pubkey) + if keystore.isSome(): + # Just update validator's `data` field with new data from keystore. + validator.data = keystore.get() + else: + deleteValidators.add(validator.pubkey) + + for pubkey in deleteValidators: + pool[].removeValidator(pubkey) + + # Adding new dynamic keystores. + for keystore in keystores.items(): + let res = pool[].getValidator(keystore.pubkey) + if res.isSome(): + let validator = res.get() + if validator.kind != ValidatorKind.Remote or + RemoteKeystoreFlag.DynamicKeystore notin validator.data.flags: + warn "Attempt to replace local validator with dynamic remote validator", + pubkey = validator.pubkey, validator = shortLog(validator), + remote_signer = $keystore.remotes, + local_validator_kind = validator.kind + else: + addProc(keystore) + proc signWithDistributedKey(v: AttachedValidator, request: Web3SignerRequest): Future[SignatureResult] {.async.} = diff --git a/docs/the_nimbus_book/src/options.md b/docs/the_nimbus_book/src/options.md index 8c998f7d6..772044acc 100644 --- a/docs/the_nimbus_book/src/options.md +++ b/docs/the_nimbus_book/src/options.md @@ -34,6 +34,7 @@ The following options are available: -d, --data-dir The directory where nimbus will store all blockchain data. --validators-dir A directory containing validator keystores. --validators-source Remote Web3Signer URL that will be used as a source of validators. + --validators-source-interval Number of minutes between validator list updates [=60]. --secrets-dir A directory containing validator keystore passwords. --wallets-dir A directory containing wallet files. --web3-url One or more execution layer Engine API URLs. diff --git a/tests/consensus_spec/altair/test_fixture_rewards.nim b/tests/consensus_spec/altair/test_fixture_rewards.nim index 7897a6039..b9e464534 100644 --- a/tests/consensus_spec/altair/test_fixture_rewards.nim +++ b/tests/consensus_spec/altair/test_fixture_rewards.nim @@ -9,8 +9,8 @@ import # Beacon chain internals - ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch], - ../../beacon_chain/spec/datatypes/altair, + ../../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch], + ../../../beacon_chain/spec/datatypes/altair, # Test utilities ../../testutil, ../fixtures_utils, ../os_ops diff --git a/tests/consensus_spec/altair/test_fixture_ssz_consensus_objects.nim b/tests/consensus_spec/altair/test_fixture_ssz_consensus_objects.nim index c3facfe51..83e8062d9 100644 --- a/tests/consensus_spec/altair/test_fixture_ssz_consensus_objects.nim +++ b/tests/consensus_spec/altair/test_fixture_ssz_consensus_objects.nim @@ -14,7 +14,7 @@ import # Third-party yaml, # Beacon chain internals - ../../beacon_chain/spec/datatypes/altair, + ../../../beacon_chain/spec/datatypes/altair, # Status libraries snappy, # Test utilities diff --git a/tests/consensus_spec/bellatrix/test_fixture_operations.nim b/tests/consensus_spec/bellatrix/test_fixture_operations.nim index cec248fb6..b7ec7945b 100644 --- a/tests/consensus_spec/bellatrix/test_fixture_operations.nim +++ b/tests/consensus_spec/bellatrix/test_fixture_operations.nim @@ -126,17 +126,19 @@ suite baseDescription & "Deposit " & preset(): OpDepositsDir, suiteName, "Deposit", "deposit", applyDeposit, path) suite baseDescription & "Execution Payload " & preset(): - for path in walkTests(OpExecutionPayloadDir): - proc applyExecutionPayload( + proc makeApplyExecutionPayloadCb(path: string): auto = + return proc( preState: var bellatrix.BeaconState, body: bellatrix.BeaconBlockBody): Result[void, cstring] = - let payloadValid = - os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml"). - contains("execution_valid: true") + let payloadValid = os_ops.readFile( + OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml" + ).contains("execution_valid: true") func executePayload(_: bellatrix.ExecutionPayload): bool = payloadValid process_execution_payload( preState, body.execution_payload, executePayload) + for path in walkTests(OpExecutionPayloadDir): + let applyExecutionPayload = makeApplyExecutionPayloadCb(path) runTest[bellatrix.BeaconBlockBody, typeof applyExecutionPayload]( OpExecutionPayloadDir, suiteName, "Execution Payload", "body", applyExecutionPayload, path) diff --git a/tests/consensus_spec/bellatrix/test_fixture_rewards.nim b/tests/consensus_spec/bellatrix/test_fixture_rewards.nim index f1aa1153e..c25ebe073 100644 --- a/tests/consensus_spec/bellatrix/test_fixture_rewards.nim +++ b/tests/consensus_spec/bellatrix/test_fixture_rewards.nim @@ -9,8 +9,8 @@ import # Beacon chain internals - ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch], - ../../beacon_chain/spec/datatypes/[altair, bellatrix], + ../../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch], + ../../../beacon_chain/spec/datatypes/[altair, bellatrix], # Test utilities ../../testutil, ../fixtures_utils, ../os_ops diff --git a/tests/consensus_spec/bellatrix/test_fixture_ssz_consensus_objects.nim b/tests/consensus_spec/bellatrix/test_fixture_ssz_consensus_objects.nim index 4234731d5..1a47724f8 100644 --- a/tests/consensus_spec/bellatrix/test_fixture_ssz_consensus_objects.nim +++ b/tests/consensus_spec/bellatrix/test_fixture_ssz_consensus_objects.nim @@ -11,7 +11,7 @@ import # Third-party yaml, # Beacon chain internals - ../../beacon_chain/spec/datatypes/[altair, bellatrix], + ../../../beacon_chain/spec/datatypes/[altair, bellatrix], # Status libraries snappy, # Test utilities diff --git a/tests/consensus_spec/capella/test_fixture_operations.nim b/tests/consensus_spec/capella/test_fixture_operations.nim index b5cff97e6..17621de41 100644 --- a/tests/consensus_spec/capella/test_fixture_operations.nim +++ b/tests/consensus_spec/capella/test_fixture_operations.nim @@ -143,17 +143,19 @@ suite baseDescription & "Deposit " & preset(): OpDepositsDir, suiteName, "Deposit", "deposit", applyDeposit, path) suite baseDescription & "Execution Payload " & preset(): - for path in walkTests(OpExecutionPayloadDir): - proc applyExecutionPayload( + proc makeApplyExecutionPayloadCb(path: string): auto = + return proc( preState: var capella.BeaconState, body: capella.BeaconBlockBody): Result[void, cstring] = - let payloadValid = - os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml"). - contains("execution_valid: true") + let payloadValid = os_ops.readFile( + OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml" + ).contains("execution_valid: true") func executePayload(_: capella.ExecutionPayload): bool = payloadValid process_execution_payload( preState, body.execution_payload, executePayload) + for path in walkTests(OpExecutionPayloadDir): + let applyExecutionPayload = makeApplyExecutionPayloadCb(path) runTest[capella.BeaconBlockBody, typeof applyExecutionPayload]( OpExecutionPayloadDir, suiteName, "Execution Payload", "body", applyExecutionPayload, path) diff --git a/tests/consensus_spec/capella/test_fixture_rewards.nim b/tests/consensus_spec/capella/test_fixture_rewards.nim index 422e1def5..b02d8770c 100644 --- a/tests/consensus_spec/capella/test_fixture_rewards.nim +++ b/tests/consensus_spec/capella/test_fixture_rewards.nim @@ -9,8 +9,8 @@ import # Beacon chain internals - ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch], - ../../beacon_chain/spec/datatypes/[altair, capella], + ../../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch], + ../../../beacon_chain/spec/datatypes/[altair, capella], # Test utilities ../../testutil, ../fixtures_utils, ../os_ops diff --git a/tests/consensus_spec/capella/test_fixture_ssz_consensus_objects.nim b/tests/consensus_spec/capella/test_fixture_ssz_consensus_objects.nim index f54401ef1..0d04766fa 100644 --- a/tests/consensus_spec/capella/test_fixture_ssz_consensus_objects.nim +++ b/tests/consensus_spec/capella/test_fixture_ssz_consensus_objects.nim @@ -14,13 +14,13 @@ import # Third-party yaml, # Beacon chain internals - ../../beacon_chain/spec/datatypes/[altair, capella], + ../../../beacon_chain/spec/datatypes/[altair, capella], # Status libraries snappy, # Test utilities ../../testutil, ../fixtures_utils, ../os_ops -from ../../beacon_chain/spec/datatypes/bellatrix import PowBlock +from ../../../beacon_chain/spec/datatypes/bellatrix import PowBlock # SSZ tests of consensus objects (minimal/mainnet preset specific) diff --git a/tests/consensus_spec/deneb/test_fixture_operations.nim b/tests/consensus_spec/deneb/test_fixture_operations.nim index 7daeceafd..9ef35b03c 100644 --- a/tests/consensus_spec/deneb/test_fixture_operations.nim +++ b/tests/consensus_spec/deneb/test_fixture_operations.nim @@ -146,16 +146,18 @@ suite baseDescription & "Deposit " & preset(): OpDepositsDir, suiteName, "Deposit", "deposit", applyDeposit, path) suite baseDescription & "Execution Payload " & preset(): - for path in walkTests(OpExecutionPayloadDir): - proc applyExecutionPayload( + proc makeApplyExecutionPayloadCb(path: string): auto = + return proc( preState: var deneb.BeaconState, body: deneb.BeaconBlockBody): Result[void, cstring] = - let payloadValid = - os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml"). - contains("execution_valid: true") + let payloadValid = os_ops.readFile( + OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml" + ).contains("execution_valid: true") func executePayload(_: deneb.ExecutionPayload): bool = payloadValid process_execution_payload(preState, body, executePayload) + for path in walkTests(OpExecutionPayloadDir): + let applyExecutionPayload = makeApplyExecutionPayloadCb(path) runTest[deneb.BeaconBlockBody, typeof applyExecutionPayload]( OpExecutionPayloadDir, suiteName, "Execution Payload", "body", applyExecutionPayload, path) diff --git a/tests/consensus_spec/deneb/test_fixture_rewards.nim b/tests/consensus_spec/deneb/test_fixture_rewards.nim index 390771286..fbbfc8c38 100644 --- a/tests/consensus_spec/deneb/test_fixture_rewards.nim +++ b/tests/consensus_spec/deneb/test_fixture_rewards.nim @@ -9,8 +9,8 @@ import # Beacon chain internals - ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch], - ../../beacon_chain/spec/datatypes/[altair, deneb], + ../../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch], + ../../../beacon_chain/spec/datatypes/[altair, deneb], # Test utilities ../../testutil, ../fixtures_utils, ../os_ops diff --git a/tests/consensus_spec/deneb/test_fixture_ssz_consensus_objects.nim b/tests/consensus_spec/deneb/test_fixture_ssz_consensus_objects.nim index 149fb6487..7986f7aae 100644 --- a/tests/consensus_spec/deneb/test_fixture_ssz_consensus_objects.nim +++ b/tests/consensus_spec/deneb/test_fixture_ssz_consensus_objects.nim @@ -14,14 +14,14 @@ import # Third-party yaml, # Beacon chain internals - ../../beacon_chain/spec/datatypes/[altair, deneb], + ../../../beacon_chain/spec/datatypes/[altair, deneb], # Status libraries snappy, # Test utilities ../../testutil, ../fixtures_utils, ../os_ops -from ../../beacon_chain/spec/datatypes/bellatrix import PowBlock -from ../../beacon_chain/spec/datatypes/capella import +from ../../../beacon_chain/spec/datatypes/bellatrix import PowBlock +from ../../../beacon_chain/spec/datatypes/capella import BLSToExecutionChange, SignedBLSToExecutionChange, HistoricalSummary, Withdrawal diff --git a/tests/consensus_spec/phase0/test_fixture_rewards.nim b/tests/consensus_spec/phase0/test_fixture_rewards.nim index a32441fe1..2ea3d5df1 100644 --- a/tests/consensus_spec/phase0/test_fixture_rewards.nim +++ b/tests/consensus_spec/phase0/test_fixture_rewards.nim @@ -10,8 +10,8 @@ import # Standard library # Beacon chain internals - ../../beacon_chain/spec/[validator, helpers, state_transition_epoch], - ../../beacon_chain/spec/datatypes/phase0, + ../../../beacon_chain/spec/[validator, helpers, state_transition_epoch], + ../../../beacon_chain/spec/datatypes/phase0, # Test utilities ../../testutil, ../fixtures_utils, ../os_ops diff --git a/tests/consensus_spec/phase0/test_fixture_ssz_consensus_objects.nim b/tests/consensus_spec/phase0/test_fixture_ssz_consensus_objects.nim index 7e4a49a78..b63772b58 100644 --- a/tests/consensus_spec/phase0/test_fixture_ssz_consensus_objects.nim +++ b/tests/consensus_spec/phase0/test_fixture_ssz_consensus_objects.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2018-2022 Status Research & Development GmbH +# Copyright (c) 2018-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -14,7 +14,7 @@ import # Third-party yaml, # Beacon chain internals - ../../beacon_chain/spec/datatypes/phase0, + ../../../beacon_chain/spec/datatypes/phase0, # Status libraries snappy, # Test utilities diff --git a/tests/consensus_spec/test_fixture_light_client_single_merkle_proof.nim b/tests/consensus_spec/test_fixture_light_client_single_merkle_proof.nim index 6926dc8c8..4b55fc50b 100644 --- a/tests/consensus_spec/test_fixture_light_client_single_merkle_proof.nim +++ b/tests/consensus_spec/test_fixture_light_client_single_merkle_proof.nim @@ -15,7 +15,7 @@ import # Third-party yaml, # Beacon chain internals - ../../../beacon_chain/spec/helpers, + ../../beacon_chain/spec/helpers, # Test utilities ../testutil, ./fixtures_utils, ./os_ops diff --git a/tests/consensus_spec/test_fixture_light_client_sync.nim b/tests/consensus_spec/test_fixture_light_client_sync.nim index 5eb943053..f493f294c 100644 --- a/tests/consensus_spec/test_fixture_light_client_sync.nim +++ b/tests/consensus_spec/test_fixture_light_client_sync.nim @@ -15,7 +15,7 @@ import # Third-party yaml, # Beacon chain internals - ../../../beacon_chain/spec/[forks, light_client_sync], + ../../beacon_chain/spec/[forks, light_client_sync], # Test utilities ../testutil, ./fixtures_utils, ./os_ops diff --git a/tests/consensus_spec/test_fixture_light_client_update_ranking.nim b/tests/consensus_spec/test_fixture_light_client_update_ranking.nim index ca2496573..e60d6db92 100644 --- a/tests/consensus_spec/test_fixture_light_client_update_ranking.nim +++ b/tests/consensus_spec/test_fixture_light_client_update_ranking.nim @@ -15,7 +15,7 @@ import # Third-party yaml, # Beacon chain internals - ../../../beacon_chain/spec/helpers, + ../../beacon_chain/spec/helpers, # Test utilities ../testutil, ./fixtures_utils, ./os_ops diff --git a/tests/consensus_spec/test_fixture_sanity_blocks.nim b/tests/consensus_spec/test_fixture_sanity_blocks.nim index 48a6bc92a..8ad50c9db 100644 --- a/tests/consensus_spec/test_fixture_sanity_blocks.nim +++ b/tests/consensus_spec/test_fixture_sanity_blocks.nim @@ -15,9 +15,9 @@ import ../testutil from std/sequtils import toSeq -from ../../../beacon_chain/spec/forks import +from ../../beacon_chain/spec/forks import ForkedEpochInfo, ForkedHashedBeaconState, fromSszBytes, getStateRoot, new -from ../../../beacon_chain/spec/presets import +from ../../beacon_chain/spec/presets import const_preset, defaultRuntimeConfig from ./fixtures_utils import SSZ, SszTestsDir, hash_tree_root, parseTest, readSszBytes, toSszType @@ -92,22 +92,22 @@ template runForkBlockTests( runForkBlockTests( "phase0", "Phase 0", phase0.BeaconState, phase0.SignedBeaconBlock) -from ../../../beacon_chain/spec/datatypes/altair import +from ../../beacon_chain/spec/datatypes/altair import BeaconState, SignedBeaconBlock runForkBlockTests( "altair", "Altair", altair.BeaconState, altair.SignedBeaconBlock) -from ../../../beacon_chain/spec/datatypes/bellatrix import +from ../../beacon_chain/spec/datatypes/bellatrix import BeaconState, SignedBeaconBlock runForkBlockTests( "bellatrix", "Bellatrix", bellatrix.BeaconState, bellatrix.SignedBeaconBlock) -from ../../../beacon_chain/spec/datatypes/capella import +from ../../beacon_chain/spec/datatypes/capella import BeaconState, SignedBeaconBlock runForkBlockTests( "capella", "Capella", capella.BeaconState, capella.SignedBeaconBlock) -from ../../../beacon_chain/spec/datatypes/deneb import +from ../../beacon_chain/spec/datatypes/deneb import BeaconState, SignedBeaconBlock runForkBlockTests( "deneb", "Deneb", deneb.BeaconState, deneb.SignedBeaconBlock) diff --git a/tests/consensus_spec/test_fixture_sanity_slots.nim b/tests/consensus_spec/test_fixture_sanity_slots.nim index f56521190..7cc334c3f 100644 --- a/tests/consensus_spec/test_fixture_sanity_slots.nim +++ b/tests/consensus_spec/test_fixture_sanity_slots.nim @@ -70,7 +70,7 @@ suite "EF - Bellatrix - Sanity - Slots " & preset(): bellatrixSanitySlotsDir, relative = true, checkDir = true): runTest(bellatrix.BeaconState, bellatrixSanitySlotsDir, "Bellatrix", suiteName, path) -from ../../../beacon_chain/spec/datatypes/capella import BeaconState +from ../../beacon_chain/spec/datatypes/capella import BeaconState suite "EF - Capella - Sanity - Slots " & preset(): const capellaSanitySlotsDir = sanitySlotsDir("capella") @@ -78,7 +78,7 @@ suite "EF - Capella - Sanity - Slots " & preset(): capellaSanitySlotsDir, relative = true, checkDir = true): runTest(capella.BeaconState, capellaSanitySlotsDir, "Capella", suiteName, path) -from ../../../beacon_chain/spec/datatypes/deneb import BeaconState +from ../../beacon_chain/spec/datatypes/deneb import BeaconState suite "EF - Deneb - Sanity - Slots " & preset(): const denebSanitySlotsDir = sanitySlotsDir("deneb") diff --git a/tests/test_deposit_snapshots.nim b/tests/test_deposit_snapshots.nim index 6128018a5..2f2e9eecc 100644 --- a/tests/test_deposit_snapshots.nim +++ b/tests/test_deposit_snapshots.nim @@ -10,8 +10,8 @@ import std/[os, random, strutils, times], chronos, stew/results, unittest2, chronicles, - ../../beacon_chain/beacon_chain_db, - ../../beacon_chain/spec/deposit_snapshots + ../beacon_chain/beacon_chain_db, + ../beacon_chain/spec/deposit_snapshots from eth/db/kvstore import kvStore from nimcrypto import toDigest diff --git a/tests/test_helpers.nim b/tests/test_helpers.nim index 1e80c447e..10747697c 100644 --- a/tests/test_helpers.nim +++ b/tests/test_helpers.nim @@ -16,7 +16,7 @@ import ../beacon_chain/spec/[forks, helpers, state_transition], ../beacon_chain/spec/datatypes/[bellatrix, capella], # Test utilities - ./unittest2, mocking/mock_genesis + unittest2, mocking/mock_genesis suite "Spec helpers": test "integer_squareroot": diff --git a/tests/test_validator_pool.nim b/tests/test_validator_pool.nim index 76c2438fc..fa6f923d8 100644 --- a/tests/test_validator_pool.nim +++ b/tests/test_validator_pool.nim @@ -1,5 +1,5 @@ # beacon_chain -# Copyright (c) 2022 Status Research & Development GmbH +# Copyright (c) 2022-2023 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). @@ -8,8 +8,26 @@ {.used.} import - unittest2, - ../beacon_chain/validators/validator_pool + std/[algorithm, sequtils], + chronos/unittest2/asynctests, + presto, confutils, + ../beacon_chain/validators/[validator_pool, keystore_management], + ../beacon_chain/[conf, beacon_node] + +func createPubKey(number: int8): ValidatorPubKey = + var res = ValidatorPubKey() + res.blob[0] = uint8(number) + res + +func createLocal(pubkey: ValidatorPubKey): KeystoreData = + KeystoreData(kind: KeystoreKind.Local, pubkey: pubkey) + +func createRemote(pubkey: ValidatorPubKey): KeystoreData = + KeystoreData(kind: KeystoreKind.Remote, pubkey: pubkey) + +func createDynamic(pubkey: ValidatorPubKey): KeystoreData = + KeystoreData(kind: KeystoreKind.Remote, pubkey: pubkey, + flags: {RemoteKeystoreFlag.DynamicKeystore}) func makeValidatorAndIndex( index: ValidatorIndex, activation_epoch: Epoch): Opt[ValidatorAndIndex] = @@ -18,6 +36,33 @@ func makeValidatorAndIndex( validator: Validator(activation_epoch: activation_epoch) ) +func cmp(a, b: array[48, byte]): int = + for index, ch in a.pairs(): + if ch < b[index]: + return -1 + elif ch > b[index]: + return 1 + 0 + +func cmp(a, b: KeystoreData): int = + if (a.kind == b.kind) and (a.pubkey == b.pubkey): + if a.kind == KeystoreKind.Remote: + if a.flags == b.flags: + 0 + else: + card(a.flags) - card(b.flags) + else: + 0 + else: + cmp(a.pubkey.blob, b.pubkey.blob) + +func checkResponse(a, b: openArray[KeystoreData]): bool = + if len(a) != len(b): return false + for index, item in a.pairs(): + if cmp(item, b[index]) != 0: + return false + true + suite "Validator pool": test "Doppelganger for genesis validator": let @@ -80,3 +125,171 @@ suite "Validator pool": not v.doppelgangerReady(GENESIS_EPOCH.start_slot) v.doppelgangerReady(now) + + asyncTest "Dynamic validator set: queryValidatorsSource() test": + proc makeJson(keys: openArray[ValidatorPubKey]): string = + var res = "[" + res.add(keys.mapIt("\"0x" & it.toHex() & "\"").join(",")) + res.add("]") + res + + var testStage = 0 + proc testValidate(pattern: string, value: string): int = 0 + var router = RestRouter.init(testValidate) + router.api(MethodGet, "/api/v1/eth2/publicKeys") do () -> RestApiResponse: + case testStage + of 0: + let data = [createPubKey(1), createPubKey(2)].makeJson() + return RestApiResponse.response(data, Http200, "application/json") + of 1: + let data = [createPubKey(1)].makeJson() + return RestApiResponse.response(data, Http200, "application/json") + of 2: + var data: seq[ValidatorPubKey] + return RestApiResponse.response(data.makeJson(), Http200, + "application/json") + else: + return RestApiResponse.response("INCORRECT TEST STAGE", Http400, + "text/plain") + + var sres = RestServerRef.new(router, initTAddress("127.0.0.1:0")) + let + server = sres.get() + serverAddress = server.server.instance.localAddress() + config = + try: + BeaconNodeConf.load(cmdLine = + mapIt(["--validators-source=http://" & $serverAddress], it)) + except Exception as exc: + raiseAssert exc.msg + + server.start() + try: + block: + testStage = 0 + let res = await queryValidatorsSource(config) + check: + res.isOk() + checkResponse( + res.get(), + [createDynamic(createPubKey(1)), createDynamic(createPubKey(2))]) + block: + testStage = 1 + let res = await queryValidatorsSource(config) + check: + res.isOk() + checkResponse(res.get(), [createDynamic(createPubKey(1))]) + block: + testStage = 2 + let res = await queryValidatorsSource(config) + check: + res.isOk() + len(res.get()) == 0 + block: + testStage = 3 + let res = await queryValidatorsSource(config) + check: + res.isErr() + finally: + await server.closeWait() + + test "Dynamic validator set: updateDynamicValidators() test": + let + fee = default(Eth1Address) + gas = 30000000'u64 + + proc checkPool(pool: ValidatorPool, expected: openArray[KeystoreData]) = + let + attachedKeystores = + block: + var res: seq[KeystoreData] + for validator in pool: + res.add(validator.data) + sorted(res, cmp) + sortedExpected = sorted(expected, cmp) + + for index, value in attachedKeystores: + check cmp(value, sortedExpected[index]) == 0 + + var pool = (ref ValidatorPool)() + discard pool[].addValidator(createLocal(createPubKey(1)), fee, gas) + discard pool[].addValidator(createRemote(createPubKey(2)), fee, gas) + discard pool[].addValidator(createDynamic(createPubKey(3)), fee, gas) + + proc addValidator(data: KeystoreData) {.gcsafe.} = + discard pool[].addValidator(data, fee, gas) + + # Adding new dynamic keystores. + block: + let + expected = [ + createLocal(createPubKey(1)), + createRemote(createPubKey(2)), + createDynamic(createPubKey(3)), + createDynamic(createPubKey(4)), + createDynamic(createPubKey(5)) + ] + keystores = [ + createDynamic(createPubKey(3)), + createDynamic(createPubKey(4)), + createDynamic(createPubKey(5)) + ] + pool.updateDynamicValidators(keystores, addValidator) + pool[].checkPool(expected) + + # Removing dynamic keystores. + block: + let + expected = [ + createLocal(createPubKey(1)), + createRemote(createPubKey(2)), + createDynamic(createPubKey(3)) + ] + keystores = [ + createDynamic(createPubKey(3)), + ] + pool.updateDynamicValidators(keystores, addValidator) + pool[].checkPool(expected) + + # Adding and removing keystores at same time. + block: + let + expected = [ + createLocal(createPubKey(1)), + createRemote(createPubKey(2)), + createDynamic(createPubKey(4)), + createDynamic(createPubKey(5)) + ] + keystores = [ + createDynamic(createPubKey(4)), + createDynamic(createPubKey(5)) + ] + pool.updateDynamicValidators(keystores, addValidator) + pool[].checkPool(expected) + + # Adding dynamic keystores with keys which are static. + block: + let + expected = [ + createLocal(createPubKey(1)), + createRemote(createPubKey(2)), + createDynamic(createPubKey(3)) + ] + keystores = [ + createDynamic(createPubKey(1)), + createDynamic(createPubKey(2)), + createDynamic(createPubKey(3)), + ] + pool.updateDynamicValidators(keystores, addValidator) + pool[].checkPool(expected) + + # Empty response + block: + let + expected = [ + createLocal(createPubKey(1)), + createRemote(createPubKey(2)) + ] + var keystores: seq[KeystoreData] + pool.updateDynamicValidators(keystores, addValidator) + pool[].checkPool(expected) diff --git a/vendor/nim-ssz-serialization b/vendor/nim-ssz-serialization index 79a30ed94..5b7d6d065 160000 --- a/vendor/nim-ssz-serialization +++ b/vendor/nim-ssz-serialization @@ -1 +1 @@ -Subproject commit 79a30ed946d1514c3a123540e1326108f4e4b48f +Subproject commit 5b7d6d0654b0a34004580c484e2a87eaac50725e