Merge commit '99507e19cde9ae5694bd98fe5170c07e9704232b' into dev/etan/lc-wasm4

AllTests-mainnet.md

@@ -604,8 +604,10 @@ OK: 6/6 Fail: 0/6 Skip: 0/6
 ```diff
 + Doppelganger for genesis validator                                                         OK
 + Doppelganger for validator that activates in same epoch as check                           OK
++ Dynamic validator set: queryValidatorsSource() test                                        OK
++ Dynamic validator set: updateDynamicValidators() test                                      OK
 ```
-OK: 2/2 Fail: 0/2 Skip: 0/2
+OK: 4/4 Fail: 0/4 Skip: 0/4
 ## Zero signature sanity checks
 ```diff
 + SSZ serialization roundtrip of SignedBeaconBlockHeader                                     OK
@@ -700,4 +702,4 @@ OK: 2/2 Fail: 0/2 Skip: 0/2
 OK: 9/9 Fail: 0/9 Skip: 0/9
 
 ---TOTAL---
-OK: 393/398 Fail: 0/398 Skip: 5/398
+OK: 395/400 Fail: 0/400 Skip: 5/400

beacon_chain/conf.nim

@@ -37,7 +37,8 @@ export
   defaultEth2TcpPort, enabledLogLevel, ValidIpAddress,
   defs, parseCmdArg, completeCmdArg, network_metadata,
   el_conf, network, BlockHashOrNumber,
-  confTomlDefs, confTomlNet, confTomlUri
+  confTomlDefs, confTomlNet, confTomlUri,
+  LightClientDataImportMode
 
 declareGauge network_name, "network name", ["name"]
 
@@ -165,6 +166,11 @@ type
       desc: "Remote Web3Signer URL that will be used as a source of validators"
       name: "validators-source"}: Option[string]
 
+    validatorsSourceInverval* {.
+      desc: "Number of minutes between validator list updates"
+      name: "validators-source-interval"
+      defaultValue: 60 .}: Natural
+
     secretsDirFlag* {.
       desc: "A directory containing validator keystore passwords"
       name: "secrets-dir" .}: Option[InputDir]
@@ -883,6 +889,11 @@ type
       desc: "Remote Web3Signer URL that will be used as a source of validators"
       name: "validators-source"}: Option[string]
 
+    validatorsSourceInverval* {.
+      desc: "Number of minutes between validator list updates"
+      name: "validators-source-interval"
+      defaultValue: 60 .}: Natural
+
     secretsDirFlag* {.
       desc: "A directory containing validator keystore passwords"
       name: "secrets-dir" .}: Option[InputDir]

beacon_chain/deposits.nim

@@ -8,7 +8,7 @@
 
 import
   std/[os, sequtils, times],
-  stew/byteutils,
+  stew/[byteutils, base10],
   chronicles,
   ./spec/eth2_apis/rest_beacon_client,
   ./spec/signatures,
@@ -220,15 +220,29 @@ proc restValidatorExit(config: BeaconNodeConf) {.async.} =
     quit 1
 
   let signingFork = try:
-    let response = await client.getSpec()
+    let response = await client.getSpecVC()
     if response.status == 200:
-      let spec = response.data
+      let
+        spec = response.data.data
+        denebForkEpoch =
+          block:
+            let s = spec.getOrDefault("DENEB_FORK_EPOCH", $FAR_FUTURE_EPOCH)
+            Epoch(Base10.decode(uint64, s).get(uint64(FAR_FUTURE_EPOCH)))
       # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/beacon-chain.md#voluntary-exits
       # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/deneb/beacon-chain.md#modified-process_voluntary_exit
-      if currentEpoch >= Epoch(spec.data.DENEB_FORK_EPOCH):
+      if currentEpoch >= denebForkEpoch:
+        let capellaForkVersion =
+          block:
+            var res: Version
+            # CAPELLA_FOR_VERSION has specific format - "0x01000000", so
+            # default empty string is invalid, so `hexToByteArrayStrict`
+            # will raise exception on empty string.
+            let s = spec.getOrDefault("CAPELLA_FORK_VERSION", "")
+            hexToByteArrayStrict(s, distinctBase(res))
+            res
         Fork(
-          current_version: spec.data.CAPELLA_FORK_VERSION,
+          current_version: capellaForkVersion,
-          previous_version: spec.data.CAPELLA_FORK_VERSION,
+          previous_version: capellaForkVersion,
           epoch: GENESIS_EPOCH)  # irrelevant when current/previous identical
       else:
         fork
@@ -239,6 +253,8 @@ proc restValidatorExit(config: BeaconNodeConf) {.async.} =
            reason = exc.msg
     quit 1
 
+  debug "Signing fork obtained", fork = fork
+
   if not config.printData:
     case askForExitConfirmation()
     of ClientExitAction.abort:
@@ -249,7 +265,8 @@ proc restValidatorExit(config: BeaconNodeConf) {.async.} =
   var hadErrors = false
   for validator in validators:
     let restValidator = try:
-      let response = await client.getStateValidatorPlain(stateIdHead, validator.getIdent)
+      let response = await client.getStateValidatorPlain(
+        stateIdHead, validator.getIdent)
       if response.status == 200:
         let validatorInfo = decodeBytes(GetStateValidatorResponse,
                                         response.data, response.contentType)

beacon_chain/gossip_processing/batch_validation.nim

@@ -228,6 +228,13 @@ proc batchVerifyTask(task: ptr BatchTask) {.nimcall.} =
 
   discard task[].signal.fireSync()
 
+proc spawnBatchVerifyTask(tp: Taskpool, task: ptr BatchTask) =
+  # Inlining this `proc` leads to compilation problems on Nim 2.0
+  # - Error: cannot generate destructor for generic type: Isolated
+  # Workaround: Ensure that `tp.spawn` is not used within an `{.async.}` proc
+  # Possibly related to: https://github.com/nim-lang/Nim/issues/22305
+  tp.spawn batchVerifyTask(task)
+
 proc batchVerifyAsync*(
     verifier: ref BatchVerifier, signal: ThreadSignalPtr,
     batch: ref Batch): Future[bool] {.async.} =
@@ -245,7 +252,7 @@ proc batchVerifyAsync*(
   let taskPtr = addr task
   doAssert verifier[].taskpool.numThreads > 1,
     "Must have at least one separate thread or signal will never be fired"
-  verifier[].taskpool.spawn batchVerifyTask(taskPtr)
+  verifier[].taskpool.spawnBatchVerifyTask(taskPtr)
   await signal.wait()
   task.ok.load()
 

beacon_chain/nimbus_beacon_node.nim

@@ -1617,6 +1617,7 @@ proc run(node: BeaconNode) {.raises: [CatchableError].} =
 
   waitFor node.updateGossipStatus(wallSlot)
 
+  asyncSpawn pollForDynamicValidators(node)
   asyncSpawn runSlotLoop(node, wallTime, onSlotStart)
   asyncSpawn runOnSecondLoop(node)
   asyncSpawn runQueueProcessingLoop(node.blockProcessor)

beacon_chain/nimbus_validator_client.nim

@@ -90,10 +90,12 @@ proc initValidators(vc: ValidatorClientRef): Future[bool] {.async.} =
   var duplicates: seq[ValidatorPubKey]
   for keystore in listLoadableKeystores(vc.config, vc.keystoreCache):
     vc.addValidator(keystore)
-  let dynamicKeystores = await queryValidatorsSource(vc.config)
+  let res = await queryValidatorsSource(vc.config)
-  for keystore in dynamicKeystores:
+  if res.isOk():
-    vc.addValidator(keystore)
+    let dynamicKeystores = res.get()
-  return true
+    for keystore in dynamicKeystores:
+      vc.addValidator(keystore)
+  true
 
 proc initClock(vc: ValidatorClientRef): Future[BeaconClock] {.async.} =
   # This procedure performs initialization of BeaconClock using current genesis

beacon_chain/spec/eth2_apis/rest_config_calls.nim

@@ -20,10 +20,6 @@ proc getForkSchedulePlain*(): RestPlainResponse {.
      rest, endpoint: "/eth/v1/config/fork_schedule", meth: MethodGet.}
   ## https://ethereum.github.io/beacon-APIs/#/Config/getForkSchedule
 
-proc getSpec*(): RestResponse[GetSpecResponse] {.
-     rest, endpoint: "/eth/v1/config/spec", meth: MethodGet.}
-  ## https://ethereum.github.io/beacon-APIs/#/Config/getSpec
-
 proc getSpecVC*(): RestResponse[GetSpecVCResponse] {.
      rest, endpoint: "/eth/v1/config/spec", meth: MethodGet.}
   ## https://ethereum.github.io/beacon-APIs/#/Config/getSpec

beacon_chain/spec/eth2_apis/rest_types.nim

@@ -344,143 +344,6 @@ type
     of ConsensusFork.Capella:   capellaData*:   capella.BeaconBlock
     of ConsensusFork.Deneb:     denebData*:     DenebBlockContents
 
-  RestSpec* = object
-    # https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.1/presets/mainnet/phase0.yaml
-    MAX_COMMITTEES_PER_SLOT*: uint64
-    TARGET_COMMITTEE_SIZE*: uint64
-    MAX_VALIDATORS_PER_COMMITTEE*: uint64
-    SHUFFLE_ROUND_COUNT*: uint64
-    HYSTERESIS_QUOTIENT*: uint64
-    HYSTERESIS_DOWNWARD_MULTIPLIER*: uint64
-    HYSTERESIS_UPWARD_MULTIPLIER*: uint64
-    MIN_DEPOSIT_AMOUNT*: uint64
-    MAX_EFFECTIVE_BALANCE*: uint64
-    EFFECTIVE_BALANCE_INCREMENT*: uint64
-    MIN_ATTESTATION_INCLUSION_DELAY*: uint64
-    SLOTS_PER_EPOCH*: uint64
-    MIN_SEED_LOOKAHEAD*: uint64
-    MAX_SEED_LOOKAHEAD*: uint64
-    EPOCHS_PER_ETH1_VOTING_PERIOD*: uint64
-    SLOTS_PER_HISTORICAL_ROOT*: uint64
-    MIN_EPOCHS_TO_INACTIVITY_PENALTY*: uint64
-    EPOCHS_PER_HISTORICAL_VECTOR*: uint64
-    EPOCHS_PER_SLASHINGS_VECTOR*: uint64
-    HISTORICAL_ROOTS_LIMIT*: uint64
-    VALIDATOR_REGISTRY_LIMIT*: uint64
-    BASE_REWARD_FACTOR*: uint64
-    WHISTLEBLOWER_REWARD_QUOTIENT*: uint64
-    PROPOSER_REWARD_QUOTIENT*: uint64
-    INACTIVITY_PENALTY_QUOTIENT*: uint64
-    MIN_SLASHING_PENALTY_QUOTIENT*: uint64
-    PROPORTIONAL_SLASHING_MULTIPLIER*: uint64
-    MAX_PROPOSER_SLASHINGS*: uint64
-    MAX_ATTESTER_SLASHINGS*: uint64
-    MAX_ATTESTATIONS*: uint64
-    MAX_DEPOSITS*: uint64
-    MAX_VOLUNTARY_EXITS*: uint64
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/presets/mainnet/altair.yaml
-    INACTIVITY_PENALTY_QUOTIENT_ALTAIR*: uint64
-    MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR*: uint64
-    PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR*: uint64
-    SYNC_COMMITTEE_SIZE*: uint64
-    EPOCHS_PER_SYNC_COMMITTEE_PERIOD*: uint64
-    MIN_SYNC_COMMITTEE_PARTICIPANTS*: uint64
-    UPDATE_TIMEOUT*: uint64
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/presets/mainnet/bellatrix.yaml
-    INACTIVITY_PENALTY_QUOTIENT_BELLATRIX*: uint64
-    MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX*: uint64
-    PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX*: uint64
-    MAX_BYTES_PER_TRANSACTION*: uint64
-    MAX_TRANSACTIONS_PER_PAYLOAD*: uint64
-    BYTES_PER_LOGS_BLOOM*: uint64
-    MAX_EXTRA_DATA_BYTES*: uint64
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/presets/mainnet/capella.yaml
-    MAX_BLS_TO_EXECUTION_CHANGES*: uint64
-    MAX_WITHDRAWALS_PER_PAYLOAD*: uint64
-    MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP*: uint64
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.3.0/configs/mainnet.yaml
-    PRESET_BASE*: string
-    CONFIG_NAME*: string
-    TERMINAL_TOTAL_DIFFICULTY*: UInt256
-    TERMINAL_BLOCK_HASH*: BlockHash
-    TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH*: uint64
-    MIN_GENESIS_ACTIVE_VALIDATOR_COUNT*: uint64
-    MIN_GENESIS_TIME*: uint64
-    GENESIS_FORK_VERSION*: Version
-    GENESIS_DELAY*: uint64
-    ALTAIR_FORK_VERSION*: Version
-    ALTAIR_FORK_EPOCH*: uint64
-    BELLATRIX_FORK_VERSION*: Version
-    BELLATRIX_FORK_EPOCH*: uint64
-    CAPELLA_FORK_VERSION*: Version
-    CAPELLA_FORK_EPOCH*: uint64
-    DENEB_FORK_VERSION*: Version
-    DENEB_FORK_EPOCH*: uint64
-    SECONDS_PER_SLOT*: uint64
-    SECONDS_PER_ETH1_BLOCK*: uint64
-    MIN_VALIDATOR_WITHDRAWABILITY_DELAY*: uint64
-    SHARD_COMMITTEE_PERIOD*: uint64
-    ETH1_FOLLOW_DISTANCE*: uint64
-    INACTIVITY_SCORE_BIAS*: uint64
-    INACTIVITY_SCORE_RECOVERY_RATE*: uint64
-    EJECTION_BALANCE*: uint64
-    MIN_PER_EPOCH_CHURN_LIMIT*: uint64
-    CHURN_LIMIT_QUOTIENT*: uint64
-    PROPOSER_SCORE_BOOST*: uint64
-    DEPOSIT_CHAIN_ID*: uint64
-    DEPOSIT_NETWORK_ID*: uint64
-    DEPOSIT_CONTRACT_ADDRESS*: Eth1Address
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/beacon-chain.md#constants
-    # GENESIS_SLOT
-    # GENESIS_EPOCH
-    # FAR_FUTURE_EPOCH
-    # BASE_REWARDS_PER_EPOCH
-    # DEPOSIT_CONTRACT_TREE_DEPTH
-    # JUSTIFICATION_BITS_LENGTH
-    # ENDIANNESS
-    BLS_WITHDRAWAL_PREFIX*: RestWithdrawalPrefix
-    ETH1_ADDRESS_WITHDRAWAL_PREFIX*: RestWithdrawalPrefix
-    DOMAIN_BEACON_PROPOSER*: DomainType
-    DOMAIN_BEACON_ATTESTER*: DomainType
-    DOMAIN_RANDAO*: DomainType
-    DOMAIN_DEPOSIT*: DomainType
-    DOMAIN_VOLUNTARY_EXIT*: DomainType
-    DOMAIN_SELECTION_PROOF*: DomainType
-    DOMAIN_AGGREGATE_AND_PROOF*: DomainType
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/altair/beacon-chain.md#constants
-    TIMELY_SOURCE_FLAG_INDEX*: byte
-    TIMELY_TARGET_FLAG_INDEX*: byte
-    TIMELY_HEAD_FLAG_INDEX*: byte
-    TIMELY_SOURCE_WEIGHT*: uint64
-    TIMELY_TARGET_WEIGHT*: uint64
-    TIMELY_HEAD_WEIGHT*: uint64
-    SYNC_REWARD_WEIGHT*: uint64
-    PROPOSER_WEIGHT*: uint64
-    WEIGHT_DENOMINATOR*: uint64
-    DOMAIN_SYNC_COMMITTEE*: DomainType
-    DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF*: DomainType
-    DOMAIN_CONTRIBUTION_AND_PROOF*: DomainType
-    # PARTICIPATION_FLAG_WEIGHTS
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/capella/beacon-chain.md#domain-types
-    DOMAIN_BLS_TO_EXECUTION_CHANGE*: DomainType
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#constants
-    TARGET_AGGREGATORS_PER_COMMITTEE*: uint64
-    RANDOM_SUBNETS_PER_VALIDATOR*: uint64
-    EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION*: uint64
-    ATTESTATION_SUBNET_COUNT*: uint64
-
-    # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/validator.md#constants
-    TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE*: uint64
-    SYNC_COMMITTEE_SUBNET_COUNT*: uint64
-
   VCRuntimeConfig* = Table[string, string]
 
   RestDepositContract* = object
@@ -665,7 +528,6 @@ type
   GetPoolProposerSlashingsResponse* = DataEnclosedObject[seq[ProposerSlashing]]
   GetPoolVoluntaryExitsResponse* = DataEnclosedObject[seq[SignedVoluntaryExit]]
   GetProposerDutiesResponse* = DataRootEnclosedObject[seq[RestProposerDuty]]
-  GetSpecResponse* = DataEnclosedObject[RestSpec]
   GetSpecVCResponse* = DataEnclosedObject[VCRuntimeConfig]
   GetStateFinalityCheckpointsResponse* = DataEnclosedObject[RestBeaconStatesFinalityCheckpoints]
   GetStateForkResponse* = DataEnclosedObject[Fork]

beacon_chain/spec/forks.nim

@@ -254,6 +254,31 @@ type
     capella*:   ForkDigest
     deneb*:     ForkDigest
 
+template kind*(
+    x: typedesc[
+      phase0.HashedBeaconState]): ConsensusFork =
+  ConsensusFork.Phase0
+
+template kind*(
+    x: typedesc[
+      altair.HashedBeaconState]): ConsensusFork =
+  ConsensusFork.Altair
+
+template kind*(
+    x: typedesc[
+      bellatrix.HashedBeaconState]): ConsensusFork =
+  ConsensusFork.Bellatrix
+
+template kind*(
+    x: typedesc[
+      capella.HashedBeaconState]): ConsensusFork =
+  ConsensusFork.Capella
+
+template kind*(
+    x: typedesc[
+      deneb.HashedBeaconState]): ConsensusFork =
+  ConsensusFork.Deneb
+
 macro getSymbolFromForkModule(fork: static ConsensusFork,
                               symbolName: static string): untyped =
   let moduleName = case fork

beacon_chain/spec/state_transition.nim

@@ -335,10 +335,9 @@ proc state_transition*(
   state_transition_block(
     cfg, state, signedBlock, cache, flags, rollback)
 
-# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#preparing-for-a-beaconblock
 template partialBeaconBlock*(
     cfg: RuntimeConfig,
-    state: var phase0.HashedBeaconState,
+    state: var ForkyHashedBeaconState,
     proposer_index: ValidatorIndex,
     randao_reveal: ValidatorSig,
     eth1_data: Eth1Data,
@@ -347,13 +346,16 @@ template partialBeaconBlock*(
     deposits: seq[Deposit],
     validator_changes: BeaconBlockValidatorChanges,
     sync_aggregate: SyncAggregate,
-    execution_payload: bellatrix.ExecutionPayloadForSigning):
+    execution_payload: ForkyExecutionPayloadForSigning
-    phase0.BeaconBlock =
+): auto =
-  phase0.BeaconBlock(
+  const consensusFork = typeof(state).kind
+
+  # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#preparing-for-a-beaconblock
+  var res = consensusFork.BeaconBlockType(
     slot: state.data.slot,
     proposer_index: proposer_index.uint64,
     parent_root: state.latest_block_root,
-    body: phase0.BeaconBlockBody(
+    body: consensusFork.BeaconBlockBodyType(
       randao_reveal: randao_reveal,
       eth1_data: eth1data,
       graffiti: graffiti,
@@ -363,127 +365,24 @@ template partialBeaconBlock*(
       deposits: List[Deposit, Limit MAX_DEPOSITS](deposits),
       voluntary_exits: validator_changes.voluntary_exits))
 
-# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/validator.md#preparing-a-beaconblock
+  # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/validator.md#preparing-a-beaconblock
-template partialBeaconBlock*(
+  when consensusFork >= ConsensusFork.Altair:
-    cfg: RuntimeConfig,
+    res.body.sync_aggregate = sync_aggregate
-    state: var altair.HashedBeaconState,
-    proposer_index: ValidatorIndex,
-    randao_reveal: ValidatorSig,
-    eth1_data: Eth1Data,
-    graffiti: GraffitiBytes,
-    attestations: seq[Attestation],
-    deposits: seq[Deposit],
-    validator_changes: BeaconBlockValidatorChanges,
-    sync_aggregate: SyncAggregate,
-    execution_payload: bellatrix.ExecutionPayloadForSigning):
-    altair.BeaconBlock =
-  altair.BeaconBlock(
-    slot: state.data.slot,
-    proposer_index: proposer_index.uint64,
-    parent_root: state.latest_block_root,
-    body: altair.BeaconBlockBody(
-      randao_reveal: randao_reveal,
-      eth1_data: eth1data,
-      graffiti: graffiti,
-      proposer_slashings: validator_changes.proposer_slashings,
-      attester_slashings: validator_changes.attester_slashings,
-      attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations),
-      deposits: List[Deposit, Limit MAX_DEPOSITS](deposits),
-      voluntary_exits: validator_changes.voluntary_exits,
-      sync_aggregate: sync_aggregate))
 
-# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/bellatrix/validator.md#block-proposal
+  # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/bellatrix/validator.md#block-proposal
-template partialBeaconBlock*(
+  when consensusFork >= ConsensusFork.Bellatrix:
-    cfg: RuntimeConfig,
+    res.body.execution_payload = execution_payload.executionPayload
-    state: var bellatrix.HashedBeaconState,
-    proposer_index: ValidatorIndex,
-    randao_reveal: ValidatorSig,
-    eth1_data: Eth1Data,
-    graffiti: GraffitiBytes,
-    attestations: seq[Attestation],
-    deposits: seq[Deposit],
-    validator_changes: BeaconBlockValidatorChanges,
-    sync_aggregate: SyncAggregate,
-    execution_payload: bellatrix.ExecutionPayloadForSigning):
-    bellatrix.BeaconBlock =
-  bellatrix.BeaconBlock(
-    slot: state.data.slot,
-    proposer_index: proposer_index.uint64,
-    parent_root: state.latest_block_root,
-    body: bellatrix.BeaconBlockBody(
-      randao_reveal: randao_reveal,
-      eth1_data: eth1data,
-      graffiti: graffiti,
-      proposer_slashings: validator_changes.proposer_slashings,
-      attester_slashings: validator_changes.attester_slashings,
-      attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations),
-      deposits: List[Deposit, Limit MAX_DEPOSITS](deposits),
-      voluntary_exits: validator_changes.voluntary_exits,
-      sync_aggregate: sync_aggregate,
-      execution_payload: execution_payload.executionPayload))
 
-# https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/capella/validator.md#block-proposal
+  # https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/capella/validator.md#block-proposal
-template partialBeaconBlock*(
+  when consensusFork >= ConsensusFork.Capella:
-    cfg: RuntimeConfig,
+    res.body.bls_to_execution_changes =
-    state: var capella.HashedBeaconState,
+      validator_changes.bls_to_execution_changes
-    proposer_index: ValidatorIndex,
-    randao_reveal: ValidatorSig,
-    eth1_data: Eth1Data,
-    graffiti: GraffitiBytes,
-    attestations: seq[Attestation],
-    deposits: seq[Deposit],
-    validator_changes: BeaconBlockValidatorChanges,
-    sync_aggregate: SyncAggregate,
-    execution_payload: capella.ExecutionPayloadForSigning):
-    capella.BeaconBlock =
-  capella.BeaconBlock(
-    slot: state.data.slot,
-    proposer_index: proposer_index.uint64,
-    parent_root: state.latest_block_root,
-    body: capella.BeaconBlockBody(
-      randao_reveal: randao_reveal,
-      eth1_data: eth1data,
-      graffiti: graffiti,
-      proposer_slashings: validator_changes.proposer_slashings,
-      attester_slashings: validator_changes.attester_slashings,
-      attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations),
-      deposits: List[Deposit, Limit MAX_DEPOSITS](deposits),
-      voluntary_exits: validator_changes.voluntary_exits,
-      sync_aggregate: sync_aggregate,
-      execution_payload: execution_payload.executionPayload,
-      bls_to_execution_changes: validator_changes.bls_to_execution_changes))
 
-# https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/validator.md#constructing-the-beaconblockbody
+  # https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/deneb/validator.md#constructing-the-beaconblockbody
-template partialBeaconBlock*(
+  when consensusFork >= ConsensusFork.Deneb:
-    cfg: RuntimeConfig,
+    res.body.blob_kzg_commitments = execution_payload.kzgs
-    state: var deneb.HashedBeaconState,
+
-    proposer_index: ValidatorIndex,
+  res
-    randao_reveal: ValidatorSig,
-    eth1_data: Eth1Data,
-    graffiti: GraffitiBytes,
-    attestations: seq[Attestation],
-    deposits: seq[Deposit],
-    validator_changes: BeaconBlockValidatorChanges,
-    sync_aggregate: SyncAggregate,
-    execution_payload: deneb.ExecutionPayloadForSigning):
-    deneb.BeaconBlock =
-  deneb.BeaconBlock(
-    slot: state.data.slot,
-    proposer_index: proposer_index.uint64,
-    parent_root: state.latest_block_root,
-    body: deneb.BeaconBlockBody(
-      randao_reveal: randao_reveal,
-      eth1_data: eth1data,
-      graffiti: graffiti,
-      proposer_slashings: validator_changes.proposer_slashings,
-      attester_slashings: validator_changes.attester_slashings,
-      attestations: List[Attestation, Limit MAX_ATTESTATIONS](attestations),
-      deposits: List[Deposit, Limit MAX_DEPOSITS](deposits),
-      voluntary_exits: validator_changes.voluntary_exits,
-      sync_aggregate: sync_aggregate,
-      execution_payload: execution_payload.executionPayload,
-      bls_to_execution_changes: validator_changes.bls_to_execution_changes,
-      blob_kzg_commitments: execution_payload.kzgs))
 
 proc makeBeaconBlock*(
     cfg: RuntimeConfig,

beacon_chain/statediff.nim

@@ -11,19 +11,19 @@ import
   stew/assign2,
   ./spec/forks
 
-func diffModIncEpoch[T, U](hl: HashArray[U, T], startSlot: uint64):
+func diffModIncEpoch[maxLen, T](hl: HashArray[maxLen, T], startSlot: uint64):
     array[SLOTS_PER_EPOCH, T] =
-  static: doAssert U.uint64 mod SLOTS_PER_EPOCH == 0
+  static: doAssert maxLen.uint64 mod SLOTS_PER_EPOCH == 0
   doAssert startSlot mod SLOTS_PER_EPOCH == 0
   for i in startSlot ..< startSlot + SLOTS_PER_EPOCH:
-    result[i mod SLOTS_PER_EPOCH] = hl[i mod U.uint64]
+    result[i mod SLOTS_PER_EPOCH] = hl[i mod maxLen.uint64]
 
-func applyModIncrement[T, U](
+func applyModIncrement[maxLen, T](
-    ha: var HashArray[U, T], hl: array[SLOTS_PER_EPOCH, T], slot: uint64) =
+    ha: var HashArray[maxLen, T], hl: array[SLOTS_PER_EPOCH, T], slot: uint64) =
   var indexSlot = slot
 
   for item in hl:
-    ha[indexSlot mod U.uint64] = item
+    ha[indexSlot mod maxLen.uint64] = item
     indexSlot += 1
 
 func applyValidatorIdentities(
@@ -51,9 +51,9 @@ func setValidatorStatusesNoWithdrawals(
     validator[].exit_epoch = hl[i].exit_epoch
     validator[].withdrawable_epoch = hl[i].withdrawable_epoch
 
-func replaceOrAddEncodeEth1Votes[T, U](
+func replaceOrAddEncodeEth1Votes[T, maxLen](
-    votes0: openArray[T], votes0_len: int, votes1: HashList[T, U]):
+    votes0: openArray[T], votes0_len: int, votes1: HashList[T, maxLen]):
-    (bool, List[T, U]) =
+    (bool, List[T, maxLen]) =
   let
     num_votes0 = votes0.len
     lower_bound =
@@ -67,17 +67,17 @@ func replaceOrAddEncodeEth1Votes[T, U](
       else:
         num_votes0
 
-  var res = (lower_bound == 0, default(List[T, U]))
+  var res = (lower_bound == 0, default(List[T, maxLen]))
   for i in lower_bound ..< votes1.len:
     if not result[1].add votes1[i]:
       raiseAssert "same limit"
   res
 
-func replaceOrAddDecodeEth1Votes[T, U](
+func replaceOrAddDecodeEth1Votes[T, maxLen](
-    votes0: var HashList[T, U], eth1_data_votes_replaced: bool,
+    votes0: var HashList[T, maxLen], eth1_data_votes_replaced: bool,
-    votes1: List[T, U]) =
+    votes1: List[T, maxLen]) =
   if eth1_data_votes_replaced:
-    votes0 = HashList[T, U]()
+    votes0 = HashList[T, maxLen]()
 
   for item in votes1:
     if not votes0.add item:
@@ -209,7 +209,8 @@ func applyDiff*(
     state: var capella.BeaconState,
     immutableValidators: openArray[ImmutableValidatorData2],
     stateDiff: BeaconStateDiff) =
-  template assign[T, U](tgt: var HashList[T, U], src: List[T, U]) =
+  template assign[T, maxLen](
+      tgt: var HashList[T, maxLen], src: List[T, maxLen]) =
     assign(tgt.data, src)
     tgt.resetCache()
 

beacon_chain/validator_client/duties_service.nim

@@ -19,7 +19,7 @@ logScope: service = ServiceName
 type
   DutiesServiceLoop* = enum
     AttesterLoop, ProposerLoop, IndicesLoop, SyncCommitteeLoop,
-    ProposerPreparationLoop, ValidatorRegisterLoop
+    ProposerPreparationLoop, ValidatorRegisterLoop, DynamicValidatorsLoop
 
 chronicles.formatIt(DutiesServiceLoop):
   case it
@@ -29,6 +29,7 @@ chronicles.formatIt(DutiesServiceLoop):
   of SyncCommitteeLoop: "sync_committee_loop"
   of ProposerPreparationLoop: "proposer_prepare_loop"
   of ValidatorRegisterLoop: "validator_register_loop"
+  of DynamicValidatorsLoop: "dynamic_validators_loop"
 
 proc checkDuty(duty: RestAttesterDuty): bool =
   (duty.committee_length <= MAX_VALIDATORS_PER_COMMITTEE) and
@@ -588,6 +589,38 @@ proc validatorIndexLoop(service: DutiesServiceRef) {.async.} =
     await service.pollForValidatorIndices()
     await service.waitForNextSlot()
 
+proc dynamicValidatorsLoop*(service: DutiesServiceRef) {.async.} =
+  let vc = service.client
+  doAssert(vc.config.validatorsSourceInverval > 0)
+
+  proc addValidatorProc(data: KeystoreData) =
+    vc.addValidator(data)
+
+  var
+    timeout = minutes(vc.config.validatorsSourceInverval)
+    exitLoop = false
+
+  while not(exitLoop):
+    exitLoop =
+      try:
+        await sleepAsync(timeout)
+        timeout =
+          block:
+            let res = await vc.config.queryValidatorsSource()
+            if res.isOk():
+              let keystores = res.get()
+              debug "Validators source has been polled for validators",
+                    keystores_found = len(keystores),
+                    validators_source = vc.config.validatorsSource
+              vc.attachedValidators.updateDynamicValidators(keystores,
+                                                            addValidatorProc)
+              minutes(vc.config.validatorsSourceInverval)
+            else:
+              seconds(5)
+        false
+      except CancelledError:
+        true
+
 proc proposerPreparationsLoop(service: DutiesServiceRef) {.async.} =
   let vc = service.client
   debug "Beacon proposer preparation loop is waiting for initialization"
@@ -661,6 +694,12 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
         service.validatorRegisterLoop()
       else:
         nil
+    dynamicFut =
+      if vc.config.validatorsSourceInverval > 0:
+        service.dynamicValidatorsLoop()
+      else:
+        debug "Dynamic validators update loop disabled"
+        nil
 
   while true:
     # This loop could look much more nicer/better, when
@@ -673,7 +712,8 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
           FutureBase(proposeFut),
           FutureBase(indicesFut),
           FutureBase(syncFut),
-          FutureBase(prepareFut)
+          FutureBase(prepareFut),
+          FutureBase(dynamicFut)
         ]
         if not(isNil(registerFut)): futures.add(FutureBase(registerFut))
         discard await race(futures)
@@ -687,6 +727,9 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
         if not(isNil(registerFut)):
           checkAndRestart(ValidatorRegisterLoop, registerFut,
                           service.validatorRegisterLoop())
+        if not(isNil(dynamicFut)):
+          checkAndRestart(DynamicValidatorsLoop, dynamicFut,
+                          service.dynamicValidatorsLoop())
         false
       except CancelledError:
         debug "Service interrupted"
@@ -703,6 +746,8 @@ proc mainLoop(service: DutiesServiceRef) {.async.} =
           pending.add(prepareFut.cancelAndWait())
         if not(isNil(registerFut)) and not(registerFut.finished()):
           pending.add(registerFut.cancelAndWait())
+        if not(isNil(dynamicFut)) and not(dynamicFut.finished()):
+          pending.add(dynamicFut.cancelAndWait())
         if not(isNil(service.pollingAttesterDutiesTask)) and
            not(service.pollingAttesterDutiesTask.finished()):
           pending.add(service.pollingAttesterDutiesTask.cancelAndWait())

beacon_chain/validators/beacon_validators.nim

@@ -138,7 +138,12 @@ proc addValidators*(node: BeaconNode) =
 
   let dynamicStores =
     try:
-      waitFor(queryValidatorsSource(node.config))
+      let res = waitFor(queryValidatorsSource(node.config))
+      if res.isErr():
+        # Error is already reported via log warning.
+        default(seq[KeystoreData])
+      else:
+        res.get()
     except CatchableError as exc:
       warn "Unexpected error happens while polling validator's source",
            error = $exc.name, reason = $exc.msg
@@ -161,6 +166,48 @@ proc addValidators*(node: BeaconNode) =
                                                  gasLimit)
     v.updateValidator(data)
 
+proc pollForDynamicValidators*(node: BeaconNode) {.async.} =
+  if node.config.validatorsSourceInverval == 0:
+    return
+
+  proc addValidatorProc(keystore: KeystoreData) =
+    let
+      epoch = node.currentSlot().epoch
+      index = Opt.none(ValidatorIndex)
+      feeRecipient =
+        node.consensusManager[].getFeeRecipient(keystore.pubkey, index, epoch)
+      gasLimit =
+        node.consensusManager[].getGasLimit(keystore.pubkey)
+    discard node.attachedValidators[].addValidator(keystore, feeRecipient,
+                                                   gasLimit)
+
+  var
+    timeout = minutes(node.config.validatorsSourceInverval)
+    exitLoop = false
+
+  while not(exitLoop):
+    exitLoop =
+      try:
+        await sleepAsync(timeout)
+        timeout =
+          block:
+            let res = await node.config.queryValidatorsSource()
+            if res.isOk():
+              let keystores = res.get()
+              debug "Validators source has been polled for validators",
+                    keystores_found = len(keystores),
+                    validators_source = node.config.validatorsSource
+              node.attachedValidators.updateDynamicValidators(keystores,
+                                                              addValidatorProc)
+              minutes(node.config.validatorsSourceInverval)
+            else:
+              # In case of error we going to repeat our call with much smaller
+              # interval.
+              seconds(5)
+        false
+      except CancelledError:
+        true
+
 proc getValidator*(node: BeaconNode, idx: ValidatorIndex): Opt[AttachedValidator] =
   let key = ? node.dag.validatorKey(idx)
   node.attachedValidators[].getValidator(key.toPubKey())
@@ -435,11 +482,6 @@ proc makeBeaconBlockForHeadAndSlot*(
     exits = withState(state[]):
       node.validatorChangePool[].getBeaconBlockValidatorChanges(
         node.dag.cfg, forkyState.data)
-    syncAggregate =
-      if slot.epoch >= node.dag.cfg.ALTAIR_FORK_EPOCH:
-        node.syncCommitteeMsgPool[].produceSyncAggregate(head.bid, slot)
-      else:
-        SyncAggregate.init()
     payload = (await payloadFut).valueOr:
       beacon_block_production_errors.inc()
       warn "Unable to get execution payload. Skipping block proposal",
@@ -456,7 +498,7 @@ proc makeBeaconBlockForHeadAndSlot*(
       attestations,
       eth1Proposal.deposits,
       exits,
-      syncAggregate,
+      node.syncCommitteeMsgPool[].produceSyncAggregate(head.bid, slot),
       payload,
       noRollback, # Temporary state - no need for rollback
       cache,
@@ -1492,9 +1534,7 @@ proc getValidatorRegistration(
 
 proc registerValidators*(node: BeaconNode, epoch: Epoch) {.async.} =
   try:
-    if  (not node.config.payloadBuilderEnable) or
+    if not node.config.payloadBuilderEnable: return
-        node.currentSlot.epoch < node.dag.cfg.BELLATRIX_FORK_EPOCH:
-      return
 
     const HttpOk = 200
 

beacon_chain/validators/keystore_management.nim

@@ -95,6 +95,8 @@ type
   MultipleKeystoresDecryptor* = object
     previouslyUsedPassword*: string
 
+  QueryResult = Result[seq[KeystoreData], string]
+
 const
   minPasswordLen = 12
   minPasswordEntropy = 60.0
@@ -627,12 +629,11 @@ proc existsKeystore(keystoreDir: string,
       return true
   false
 
-proc queryValidatorsSource*(config: AnyConf): Future[seq[KeystoreData]] {.
+proc queryValidatorsSource*(config: AnyConf): Future[QueryResult] {.async.} =
-     async.} =
   var keystores: seq[KeystoreData]
   if config.validatorsSource.isNone() or
      len(config.validatorsSource.get()) == 0:
-    return keystores
+    return QueryResult.ok(keystores)
 
   let vsource = config.validatorsSource.get()
   logScope:
@@ -647,9 +648,9 @@ proc queryValidatorsSource*(config: AnyConf): Future[seq[KeystoreData]] {.
         let res = RestClientRef.new(vsource, prestoFlags,
                                     httpFlags, socketFlags = socketFlags)
         if res.isErr():
-          # TODO keep trying in case of temporary network failure
+          warn "Unable to resolve validator's source distributed signer " &
-          warn "Unable to resolve validator's source distributed signer address"
+               "address", reason = $res.error
-          return keystores
+          return QueryResult.err($res.error)
         res.get()
     keys =
       try:
@@ -657,26 +658,27 @@ proc queryValidatorsSource*(config: AnyConf): Future[seq[KeystoreData]] {.
         if response.status != 200:
           warn "Remote validator's source responded with error",
                error = response.status
-          return keystores
+          return QueryResult.err(
+            "Remote validator's source responded with error [" &
+              $response.status & "]")
 
         let res = decodeBytes(Web3SignerKeysResponse, response.data,
                               response.contentType)
         if res.isErr():
           warn "Unable to obtain validator's source response",
                reason = res.error
-          return keystores
+          return QueryResult.err($res.error)
-
         res.get()
       except RestError as exc:
         warn "Unable to poll validator's source", reason = $exc.msg
-        return keystores
+        return QueryResult.err($exc.msg)
       except CancelledError as exc:
         debug "The polling of validator's source was interrupted"
         raise exc
       except CatchableError as exc:
         warn "Unexpected error occured while polling validator's source",
              error = $exc.name, reason = $exc.msg
-        return keystores
+        return QueryResult.err($exc.msg)
 
   for pubkey in keys:
     keystores.add(KeystoreData(
@@ -689,7 +691,7 @@ proc queryValidatorsSource*(config: AnyConf): Future[seq[KeystoreData]] {.
       flags: {RemoteKeystoreFlag.DynamicKeystore},
       remoteType: RemoteSignerType.Web3Signer))
 
-  keystores
+  QueryResult.ok(keystores)
 
 iterator listLoadableKeys*(validatorsDir, secretsDir: string,
                            keysMask: set[KeystoreKind]): CookedPubKey =

beacon_chain/validators/validator_pool.nim

@@ -87,6 +87,8 @@ type
     slashingProtection*: SlashingProtectionDB
     doppelgangerDetectionEnabled*: bool
 
+  AddValidatorProc* = proc(keystore: KeystoreData) {.gcsafe, raises: [].}
+
 template pubkey*(v: AttachedValidator): ValidatorPubKey =
   v.data.pubkey
 
@@ -221,8 +223,12 @@ proc removeValidator*(pool: var ValidatorPool, pubkey: ValidatorPubKey) =
     of ValidatorKind.Local:
       notice "Local validator detached", pubkey, validator = shortLog(validator)
     of ValidatorKind.Remote:
-      notice "Remote validator detached", pubkey,
+      if RemoteKeystoreFlag.DynamicKeystore in validator.data.flags:
-             validator = shortLog(validator)
+        notice "Dynamic remote validator detached", pubkey,
+               validator = shortLog(validator)
+      else:
+        notice "Remote validator detached", pubkey,
+               validator = shortLog(validator)
     validators.set(pool.count().int64)
 
 func needsUpdate*(validator: AttachedValidator): bool =
@@ -373,6 +379,46 @@ func triggersDoppelganger*(
   let v = pool.getValidator(pubkey)
   v.isSome() and v[].triggersDoppelganger(epoch)
 
+proc updateDynamicValidators*(pool: ref ValidatorPool,
+                              keystores: openArray[KeystoreData],
+                              addProc: AddValidatorProc) =
+  var
+    keystoresTable: Table[ValidatorPubKey, Opt[KeystoreData]]
+    deleteValidators: seq[ValidatorPubKey]
+
+  for keystore in keystores:
+    keystoresTable[keystore.pubkey] = Opt.some(keystore)
+
+  # We preserve `Local` and `Remote` keystores which are not from dynamic set,
+  # and also we removing all the dynamic keystores which are not part of new
+  # dynamic set.
+  for validator in pool[].items():
+    if validator.kind == ValidatorKind.Remote:
+      if RemoteKeystoreFlag.DynamicKeystore in validator.data.flags:
+        let keystore = keystoresTable.getOrDefault(validator.pubkey)
+        if keystore.isSome():
+          # Just update validator's `data` field with new data from keystore.
+          validator.data = keystore.get()
+        else:
+          deleteValidators.add(validator.pubkey)
+
+  for pubkey in deleteValidators:
+    pool[].removeValidator(pubkey)
+
+  # Adding new dynamic keystores.
+  for keystore in keystores.items():
+    let res = pool[].getValidator(keystore.pubkey)
+    if res.isSome():
+      let validator = res.get()
+      if validator.kind != ValidatorKind.Remote or
+         RemoteKeystoreFlag.DynamicKeystore notin validator.data.flags:
+        warn "Attempt to replace local validator with dynamic remote validator",
+             pubkey = validator.pubkey, validator = shortLog(validator),
+             remote_signer = $keystore.remotes,
+             local_validator_kind = validator.kind
+    else:
+      addProc(keystore)
+
 proc signWithDistributedKey(v: AttachedValidator,
                             request: Web3SignerRequest): Future[SignatureResult]
                            {.async.} =

docs/the_nimbus_book/src/options.md

@@ -34,6 +34,7 @@ The following options are available:
  -d, --data-dir                The directory where nimbus will store all blockchain data.
      --validators-dir          A directory containing validator keystores.
      --validators-source       Remote Web3Signer URL that will be used as a source of validators.
+     --validators-source-interval  Number of minutes between validator list updates [=60].
      --secrets-dir             A directory containing validator keystore passwords.
      --wallets-dir             A directory containing wallet files.
      --web3-url                One or more execution layer Engine API URLs.

tests/consensus_spec/altair/test_fixture_rewards.nim

@@ -9,8 +9,8 @@
 
 import
   # Beacon chain internals
-  ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
+  ../../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
-  ../../beacon_chain/spec/datatypes/altair,
+  ../../../beacon_chain/spec/datatypes/altair,
   # Test utilities
   ../../testutil,
   ../fixtures_utils, ../os_ops

tests/consensus_spec/altair/test_fixture_ssz_consensus_objects.nim

@@ -14,7 +14,7 @@ import
   # Third-party
   yaml,
   # Beacon chain internals
-  ../../beacon_chain/spec/datatypes/altair,
+  ../../../beacon_chain/spec/datatypes/altair,
   # Status libraries
   snappy,
   # Test utilities

tests/consensus_spec/bellatrix/test_fixture_operations.nim

@@ -126,17 +126,19 @@ suite baseDescription & "Deposit " & preset():
       OpDepositsDir, suiteName, "Deposit", "deposit", applyDeposit, path)
 
 suite baseDescription & "Execution Payload " & preset():
-  for path in walkTests(OpExecutionPayloadDir):
+  proc makeApplyExecutionPayloadCb(path: string): auto =
-    proc applyExecutionPayload(
+    return proc(
         preState: var bellatrix.BeaconState, body: bellatrix.BeaconBlockBody):
         Result[void, cstring] =
-      let payloadValid =
+      let payloadValid = os_ops.readFile(
-        os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
+          OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml"
-          contains("execution_valid: true")
+        ).contains("execution_valid: true")
       func executePayload(_: bellatrix.ExecutionPayload): bool = payloadValid
       process_execution_payload(
         preState, body.execution_payload, executePayload)
 
+  for path in walkTests(OpExecutionPayloadDir):
+    let applyExecutionPayload = makeApplyExecutionPayloadCb(path)
     runTest[bellatrix.BeaconBlockBody, typeof applyExecutionPayload](
       OpExecutionPayloadDir, suiteName, "Execution Payload", "body",
       applyExecutionPayload, path)

tests/consensus_spec/bellatrix/test_fixture_rewards.nim

@@ -9,8 +9,8 @@
 
 import
   # Beacon chain internals
-  ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
+  ../../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
-  ../../beacon_chain/spec/datatypes/[altair, bellatrix],
+  ../../../beacon_chain/spec/datatypes/[altair, bellatrix],
   # Test utilities
   ../../testutil,
   ../fixtures_utils, ../os_ops

tests/consensus_spec/bellatrix/test_fixture_ssz_consensus_objects.nim

@@ -11,7 +11,7 @@ import
   # Third-party
   yaml,
   # Beacon chain internals
-  ../../beacon_chain/spec/datatypes/[altair, bellatrix],
+  ../../../beacon_chain/spec/datatypes/[altair, bellatrix],
   # Status libraries
   snappy,
   # Test utilities

tests/consensus_spec/capella/test_fixture_operations.nim

@@ -143,17 +143,19 @@ suite baseDescription & "Deposit " & preset():
       OpDepositsDir, suiteName, "Deposit", "deposit", applyDeposit, path)
 
 suite baseDescription & "Execution Payload " & preset():
-  for path in walkTests(OpExecutionPayloadDir):
+  proc makeApplyExecutionPayloadCb(path: string): auto =
-    proc applyExecutionPayload(
+    return proc(
         preState: var capella.BeaconState, body: capella.BeaconBlockBody):
         Result[void, cstring] =
-      let payloadValid =
+      let payloadValid = os_ops.readFile(
-        os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
+          OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml"
-          contains("execution_valid: true")
+        ).contains("execution_valid: true")
       func executePayload(_: capella.ExecutionPayload): bool = payloadValid
       process_execution_payload(
         preState, body.execution_payload, executePayload)
 
+  for path in walkTests(OpExecutionPayloadDir):
+    let applyExecutionPayload = makeApplyExecutionPayloadCb(path)
     runTest[capella.BeaconBlockBody, typeof applyExecutionPayload](
       OpExecutionPayloadDir, suiteName, "Execution Payload", "body",
       applyExecutionPayload, path)

tests/consensus_spec/capella/test_fixture_rewards.nim

@@ -9,8 +9,8 @@
 
 import
   # Beacon chain internals
-  ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
+  ../../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
-  ../../beacon_chain/spec/datatypes/[altair, capella],
+  ../../../beacon_chain/spec/datatypes/[altair, capella],
   # Test utilities
   ../../testutil,
   ../fixtures_utils, ../os_ops

tests/consensus_spec/capella/test_fixture_ssz_consensus_objects.nim

@@ -14,13 +14,13 @@ import
   # Third-party
   yaml,
   # Beacon chain internals
-  ../../beacon_chain/spec/datatypes/[altair, capella],
+  ../../../beacon_chain/spec/datatypes/[altair, capella],
   # Status libraries
   snappy,
   # Test utilities
   ../../testutil, ../fixtures_utils, ../os_ops
 
-from ../../beacon_chain/spec/datatypes/bellatrix import PowBlock
+from ../../../beacon_chain/spec/datatypes/bellatrix import PowBlock
 
 # SSZ tests of consensus objects (minimal/mainnet preset specific)
 

tests/consensus_spec/deneb/test_fixture_operations.nim

@@ -146,16 +146,18 @@ suite baseDescription & "Deposit " & preset():
       OpDepositsDir, suiteName, "Deposit", "deposit", applyDeposit, path)
 
 suite baseDescription & "Execution Payload " & preset():
-  for path in walkTests(OpExecutionPayloadDir):
+  proc makeApplyExecutionPayloadCb(path: string): auto =
-    proc applyExecutionPayload(
+    return proc(
         preState: var deneb.BeaconState, body: deneb.BeaconBlockBody):
         Result[void, cstring] =
-      let payloadValid =
+      let payloadValid = os_ops.readFile(
-        os_ops.readFile(OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml").
+          OpExecutionPayloadDir/"pyspec_tests"/path/"execution.yaml"
-          contains("execution_valid: true")
+        ).contains("execution_valid: true")
       func executePayload(_: deneb.ExecutionPayload): bool = payloadValid
       process_execution_payload(preState, body, executePayload)
 
+  for path in walkTests(OpExecutionPayloadDir):
+    let applyExecutionPayload = makeApplyExecutionPayloadCb(path)
     runTest[deneb.BeaconBlockBody, typeof applyExecutionPayload](
       OpExecutionPayloadDir, suiteName, "Execution Payload", "body",
       applyExecutionPayload, path)

tests/consensus_spec/deneb/test_fixture_rewards.nim

@@ -9,8 +9,8 @@
 
 import
   # Beacon chain internals
-  ../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
+  ../../../beacon_chain/spec/[beaconstate, validator, helpers, state_transition_epoch],
-  ../../beacon_chain/spec/datatypes/[altair, deneb],
+  ../../../beacon_chain/spec/datatypes/[altair, deneb],
   # Test utilities
   ../../testutil,
   ../fixtures_utils, ../os_ops

tests/consensus_spec/deneb/test_fixture_ssz_consensus_objects.nim

@@ -14,14 +14,14 @@ import
   # Third-party
   yaml,
   # Beacon chain internals
-  ../../beacon_chain/spec/datatypes/[altair, deneb],
+  ../../../beacon_chain/spec/datatypes/[altair, deneb],
   # Status libraries
   snappy,
   # Test utilities
   ../../testutil, ../fixtures_utils, ../os_ops
 
-from ../../beacon_chain/spec/datatypes/bellatrix import PowBlock
+from ../../../beacon_chain/spec/datatypes/bellatrix import PowBlock
-from ../../beacon_chain/spec/datatypes/capella import
+from ../../../beacon_chain/spec/datatypes/capella import
   BLSToExecutionChange, SignedBLSToExecutionChange, HistoricalSummary,
   Withdrawal
 

tests/consensus_spec/phase0/test_fixture_rewards.nim

@@ -10,8 +10,8 @@
 import
   # Standard library
   # Beacon chain internals
-  ../../beacon_chain/spec/[validator, helpers, state_transition_epoch],
+  ../../../beacon_chain/spec/[validator, helpers, state_transition_epoch],
-  ../../beacon_chain/spec/datatypes/phase0,
+  ../../../beacon_chain/spec/datatypes/phase0,
   # Test utilities
   ../../testutil,
   ../fixtures_utils, ../os_ops

tests/consensus_spec/phase0/test_fixture_ssz_consensus_objects.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2018-2022 Status Research & Development GmbH
+# Copyright (c) 2018-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -14,7 +14,7 @@ import
   # Third-party
   yaml,
   # Beacon chain internals
-  ../../beacon_chain/spec/datatypes/phase0,
+  ../../../beacon_chain/spec/datatypes/phase0,
   # Status libraries
   snappy,
   # Test utilities

tests/consensus_spec/test_fixture_light_client_single_merkle_proof.nim

@@ -15,7 +15,7 @@ import
   # Third-party
   yaml,
   # Beacon chain internals
-  ../../../beacon_chain/spec/helpers,
+  ../../beacon_chain/spec/helpers,
   # Test utilities
   ../testutil,
   ./fixtures_utils, ./os_ops

tests/consensus_spec/test_fixture_light_client_sync.nim

@@ -15,7 +15,7 @@ import
   # Third-party
   yaml,
   # Beacon chain internals
-  ../../../beacon_chain/spec/[forks, light_client_sync],
+  ../../beacon_chain/spec/[forks, light_client_sync],
   # Test utilities
   ../testutil,
   ./fixtures_utils, ./os_ops

tests/consensus_spec/test_fixture_light_client_update_ranking.nim

@@ -15,7 +15,7 @@ import
   # Third-party
   yaml,
   # Beacon chain internals
-  ../../../beacon_chain/spec/helpers,
+  ../../beacon_chain/spec/helpers,
   # Test utilities
   ../testutil,
   ./fixtures_utils, ./os_ops

tests/consensus_spec/test_fixture_sanity_blocks.nim

@@ -15,9 +15,9 @@ import
   ../testutil
 
 from std/sequtils import toSeq
-from ../../../beacon_chain/spec/forks import
+from ../../beacon_chain/spec/forks import
   ForkedEpochInfo, ForkedHashedBeaconState, fromSszBytes, getStateRoot, new
-from ../../../beacon_chain/spec/presets import
+from ../../beacon_chain/spec/presets import
   const_preset, defaultRuntimeConfig
 from ./fixtures_utils import
   SSZ, SszTestsDir, hash_tree_root, parseTest, readSszBytes, toSszType
@@ -92,22 +92,22 @@ template runForkBlockTests(
 runForkBlockTests(
   "phase0", "Phase 0", phase0.BeaconState, phase0.SignedBeaconBlock)
 
-from ../../../beacon_chain/spec/datatypes/altair import
+from ../../beacon_chain/spec/datatypes/altair import
   BeaconState, SignedBeaconBlock
 runForkBlockTests(
   "altair", "Altair", altair.BeaconState, altair.SignedBeaconBlock)
 
-from ../../../beacon_chain/spec/datatypes/bellatrix import
+from ../../beacon_chain/spec/datatypes/bellatrix import
   BeaconState, SignedBeaconBlock
 runForkBlockTests(
   "bellatrix", "Bellatrix", bellatrix.BeaconState, bellatrix.SignedBeaconBlock)
 
-from ../../../beacon_chain/spec/datatypes/capella import
+from ../../beacon_chain/spec/datatypes/capella import
   BeaconState, SignedBeaconBlock
 runForkBlockTests(
   "capella", "Capella", capella.BeaconState, capella.SignedBeaconBlock)
 
-from ../../../beacon_chain/spec/datatypes/deneb import
+from ../../beacon_chain/spec/datatypes/deneb import
   BeaconState, SignedBeaconBlock
 runForkBlockTests(
   "deneb", "Deneb", deneb.BeaconState, deneb.SignedBeaconBlock)

tests/consensus_spec/test_fixture_sanity_slots.nim

@@ -70,7 +70,7 @@ suite "EF - Bellatrix - Sanity - Slots " & preset():
       bellatrixSanitySlotsDir, relative = true, checkDir = true):
     runTest(bellatrix.BeaconState, bellatrixSanitySlotsDir, "Bellatrix", suiteName, path)
 
-from ../../../beacon_chain/spec/datatypes/capella import BeaconState
+from ../../beacon_chain/spec/datatypes/capella import BeaconState
 
 suite "EF - Capella - Sanity - Slots " & preset():
   const capellaSanitySlotsDir = sanitySlotsDir("capella")
@@ -78,7 +78,7 @@ suite "EF - Capella - Sanity - Slots " & preset():
       capellaSanitySlotsDir, relative = true, checkDir = true):
     runTest(capella.BeaconState, capellaSanitySlotsDir, "Capella", suiteName, path)
 
-from ../../../beacon_chain/spec/datatypes/deneb import BeaconState
+from ../../beacon_chain/spec/datatypes/deneb import BeaconState
 
 suite "EF - Deneb - Sanity - Slots " & preset():
   const denebSanitySlotsDir = sanitySlotsDir("deneb")

tests/test_deposit_snapshots.nim

@@ -10,8 +10,8 @@
 import
   std/[os, random, strutils, times],
   chronos, stew/results, unittest2, chronicles,
-  ../../beacon_chain/beacon_chain_db,
+  ../beacon_chain/beacon_chain_db,
-  ../../beacon_chain/spec/deposit_snapshots
+  ../beacon_chain/spec/deposit_snapshots
 
 from eth/db/kvstore import kvStore
 from nimcrypto import toDigest

tests/test_helpers.nim

@@ -16,7 +16,7 @@ import
   ../beacon_chain/spec/[forks, helpers, state_transition],
   ../beacon_chain/spec/datatypes/[bellatrix, capella],
   # Test utilities
-  ./unittest2, mocking/mock_genesis
+  unittest2, mocking/mock_genesis
 
 suite "Spec helpers":
   test "integer_squareroot":

tests/test_validator_pool.nim

@@ -1,5 +1,5 @@
 # beacon_chain
-# Copyright (c) 2022 Status Research & Development GmbH
+# Copyright (c) 2022-2023 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -8,8 +8,26 @@
 {.used.}
 
 import
-  unittest2,
+  std/[algorithm, sequtils],
-  ../beacon_chain/validators/validator_pool
+  chronos/unittest2/asynctests,
+  presto, confutils,
+  ../beacon_chain/validators/[validator_pool, keystore_management],
+  ../beacon_chain/[conf, beacon_node]
+
+func createPubKey(number: int8): ValidatorPubKey =
+  var res = ValidatorPubKey()
+  res.blob[0] = uint8(number)
+  res
+
+func createLocal(pubkey: ValidatorPubKey): KeystoreData =
+  KeystoreData(kind: KeystoreKind.Local, pubkey: pubkey)
+
+func createRemote(pubkey: ValidatorPubKey): KeystoreData =
+  KeystoreData(kind: KeystoreKind.Remote, pubkey: pubkey)
+
+func createDynamic(pubkey: ValidatorPubKey): KeystoreData =
+  KeystoreData(kind: KeystoreKind.Remote, pubkey: pubkey,
+               flags: {RemoteKeystoreFlag.DynamicKeystore})
 
 func makeValidatorAndIndex(
     index: ValidatorIndex, activation_epoch: Epoch): Opt[ValidatorAndIndex] =
@@ -18,6 +36,33 @@ func makeValidatorAndIndex(
     validator: Validator(activation_epoch: activation_epoch)
   )
 
+func cmp(a, b: array[48, byte]): int =
+  for index, ch in a.pairs():
+    if ch < b[index]:
+      return -1
+    elif ch > b[index]:
+      return 1
+  0
+
+func cmp(a, b: KeystoreData): int =
+  if (a.kind == b.kind) and (a.pubkey == b.pubkey):
+    if a.kind == KeystoreKind.Remote:
+      if a.flags == b.flags:
+        0
+      else:
+        card(a.flags) - card(b.flags)
+    else:
+      0
+  else:
+    cmp(a.pubkey.blob, b.pubkey.blob)
+
+func checkResponse(a, b: openArray[KeystoreData]): bool =
+  if len(a) != len(b): return false
+  for index, item in a.pairs():
+    if cmp(item, b[index]) != 0:
+      return false
+  true
+
 suite "Validator pool":
   test "Doppelganger for genesis validator":
     let
@@ -80,3 +125,171 @@ suite "Validator pool":
 
       not v.doppelgangerReady(GENESIS_EPOCH.start_slot)
       v.doppelgangerReady(now)
+
+  asyncTest "Dynamic validator set: queryValidatorsSource() test":
+    proc makeJson(keys: openArray[ValidatorPubKey]): string =
+      var res = "["
+      res.add(keys.mapIt("\"0x" & it.toHex() & "\"").join(","))
+      res.add("]")
+      res
+
+    var testStage = 0
+    proc testValidate(pattern: string, value: string): int = 0
+    var router = RestRouter.init(testValidate)
+    router.api(MethodGet, "/api/v1/eth2/publicKeys") do () -> RestApiResponse:
+      case testStage
+      of 0:
+        let data = [createPubKey(1), createPubKey(2)].makeJson()
+        return RestApiResponse.response(data, Http200, "application/json")
+      of 1:
+        let data = [createPubKey(1)].makeJson()
+        return RestApiResponse.response(data, Http200, "application/json")
+      of 2:
+        var data: seq[ValidatorPubKey]
+        return RestApiResponse.response(data.makeJson(), Http200,
+                                        "application/json")
+      else:
+        return RestApiResponse.response("INCORRECT TEST STAGE", Http400,
+                                        "text/plain")
+
+    var sres = RestServerRef.new(router, initTAddress("127.0.0.1:0"))
+    let
+      server = sres.get()
+      serverAddress = server.server.instance.localAddress()
+      config =
+        try:
+          BeaconNodeConf.load(cmdLine =
+            mapIt(["--validators-source=http://" & $serverAddress], it))
+        except Exception as exc:
+          raiseAssert exc.msg
+
+    server.start()
+    try:
+      block:
+        testStage = 0
+        let res = await queryValidatorsSource(config)
+        check:
+          res.isOk()
+          checkResponse(
+            res.get(),
+            [createDynamic(createPubKey(1)), createDynamic(createPubKey(2))])
+      block:
+        testStage = 1
+        let res = await queryValidatorsSource(config)
+        check:
+          res.isOk()
+          checkResponse(res.get(), [createDynamic(createPubKey(1))])
+      block:
+        testStage = 2
+        let res = await queryValidatorsSource(config)
+        check:
+          res.isOk()
+          len(res.get()) == 0
+      block:
+        testStage = 3
+        let res = await queryValidatorsSource(config)
+        check:
+          res.isErr()
+    finally:
+      await server.closeWait()
+
+  test "Dynamic validator set: updateDynamicValidators() test":
+    let
+      fee = default(Eth1Address)
+      gas = 30000000'u64
+
+    proc checkPool(pool: ValidatorPool, expected: openArray[KeystoreData]) =
+      let
+        attachedKeystores =
+          block:
+            var res: seq[KeystoreData]
+            for validator in pool:
+              res.add(validator.data)
+            sorted(res, cmp)
+        sortedExpected = sorted(expected, cmp)
+
+      for index, value in attachedKeystores:
+        check cmp(value, sortedExpected[index]) == 0
+
+    var pool = (ref ValidatorPool)()
+    discard pool[].addValidator(createLocal(createPubKey(1)), fee, gas)
+    discard pool[].addValidator(createRemote(createPubKey(2)), fee, gas)
+    discard pool[].addValidator(createDynamic(createPubKey(3)), fee, gas)
+
+    proc addValidator(data: KeystoreData) {.gcsafe.} =
+      discard pool[].addValidator(data, fee, gas)
+
+    # Adding new dynamic keystores.
+    block:
+      let
+        expected = [
+          createLocal(createPubKey(1)),
+          createRemote(createPubKey(2)),
+          createDynamic(createPubKey(3)),
+          createDynamic(createPubKey(4)),
+          createDynamic(createPubKey(5))
+        ]
+        keystores = [
+          createDynamic(createPubKey(3)),
+          createDynamic(createPubKey(4)),
+          createDynamic(createPubKey(5))
+        ]
+      pool.updateDynamicValidators(keystores, addValidator)
+      pool[].checkPool(expected)
+
+    # Removing dynamic keystores.
+    block:
+      let
+        expected = [
+          createLocal(createPubKey(1)),
+          createRemote(createPubKey(2)),
+          createDynamic(createPubKey(3))
+        ]
+        keystores = [
+          createDynamic(createPubKey(3)),
+        ]
+      pool.updateDynamicValidators(keystores, addValidator)
+      pool[].checkPool(expected)
+
+    # Adding and removing keystores at same time.
+    block:
+      let
+        expected = [
+          createLocal(createPubKey(1)),
+          createRemote(createPubKey(2)),
+          createDynamic(createPubKey(4)),
+          createDynamic(createPubKey(5))
+        ]
+        keystores = [
+          createDynamic(createPubKey(4)),
+          createDynamic(createPubKey(5))
+        ]
+      pool.updateDynamicValidators(keystores, addValidator)
+      pool[].checkPool(expected)
+
+    # Adding dynamic keystores with keys which are static.
+    block:
+      let
+        expected = [
+          createLocal(createPubKey(1)),
+          createRemote(createPubKey(2)),
+          createDynamic(createPubKey(3))
+        ]
+        keystores = [
+          createDynamic(createPubKey(1)),
+          createDynamic(createPubKey(2)),
+          createDynamic(createPubKey(3)),
+        ]
+      pool.updateDynamicValidators(keystores, addValidator)
+      pool[].checkPool(expected)
+
+    # Empty response
+    block:
+      let
+        expected = [
+          createLocal(createPubKey(1)),
+          createRemote(createPubKey(2))
+        ]
+      var keystores: seq[KeystoreData]
+      pool.updateDynamicValidators(keystores, addValidator)
+      pool[].checkPool(expected)

vendor/nim-ssz-serialization

@@ -1 +1 @@
-Subproject commit 79a30ed946d1514c3a123540e1326108f4e4b48f
+Subproject commit 5b7d6d0654b0a34004580c484e2a87eaac50725e