- updated the validator shell script after the keystore changes

- better logging & retrying requests on the VC side if the BN fails for some reason
- VC now fetches the attestation duties 1 epoch in advance - in the future it will tell the BN to subscribe to the appropriate attestation topics in advance based on that info
- a bunch of other code cleanup & fixes such as better naming for consoles when using multitail, etc.

reviewed in PR #1184 - proper review of the API & VC are pending

README.md

@@ -191,6 +191,8 @@ make VALIDATORS=192 NODES=6 USER_NODES=1 eth2_network_simulation
 # looks like from a single nodes' perspective.
 ```
 
+By default all validators are loaded within the beacon nodes, but if you want to use external processes as validator clients you can pass `BN_VC_VALIDATOR_SPLIT=yes` as an additional argument to the `make eth2_network_simulation` command and that will split the `VALIDATORS` between beacon nodes and validator clients - for example with `192` validators and `6` nodes you will end up with 6 beacon node and 6 validator client processes, where each of them will handle 16 validators.
+
 You can also separate the output from each beacon node in its own panel, using [multitail](http://www.vanheusden.com/multitail/):
 
 ```bash

beacon_chain/spec/eth2_apis/beacon_callsigs.nim

@@ -1,6 +1,20 @@
 import
   options,
-  ../datatypes
+  ../[datatypes, digest, crypto],
+  json_rpc/jsonmarshal,
+  callsigs_types
+
+proc get_v1_beacon_genesis(): BeaconGenesisTuple
+
+# TODO stateId is part of the REST path
+proc get_v1_beacon_states_root(stateId: string): Eth2Digest
+
+# TODO stateId is part of the REST path
+proc get_v1_beacon_states_fork(stateId: string): Fork
+
+
+
+# TODO: delete old stuff
 
 # https://github.com/ethereum/eth2.0-APIs/blob/master/apis/beacon/basic.md
 #

beacon_chain/spec/eth2_apis/callsigs_types.nim

@@ -0,0 +1,23 @@
+import
+  # Standard library
+  options,
+  # Local modules
+  # TODO for some reason "../[datatypes, digest, crypto]" results in "Error: cannot open file"
+  ../datatypes,
+  ../digest,
+  ../crypto
+
+type
+  AttesterDuties* = tuple
+    public_key: ValidatorPubKey
+    committee_index: CommitteeIndex
+    committee_length: uint64
+    validator_committee_index: uint64
+    slot: Slot
+
+  ValidatorPubkeySlotPair* = tuple[public_key: ValidatorPubKey, slot: Slot]
+
+  BeaconGenesisTuple* = tuple
+    genesis_time: uint64
+    genesis_validators_root: Eth2Digest
+    genesis_fork_version: Version

beacon_chain/spec/eth2_apis/validator_callsigs.nim

@@ -4,24 +4,17 @@ import
   # Local modules
   ../[datatypes, digest, crypto],
   json_rpc/jsonmarshal,
-  validator_callsigs_types
-
-# TODO check which arguments are part of the path in the REST API
+  callsigs_types
 
 
+# calls that return a bool are actually without a return type in the main REST API
+# spec but nim-json-rpc requires that all RPC calls have a return type.
 
-# TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
-proc get_v1_beacon_states_fork(stateId: string): Fork
-
-# TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
-proc get_v1_beacon_genesis(): BeaconGenesisTuple
-
-# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
 proc post_v1_beacon_pool_attestations(attestation: Attestation): bool
 
+# TODO slot is part of the REST path
 proc get_v1_validator_blocks(slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig): BeaconBlock
 
-# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
 proc post_v1_beacon_blocks(body: SignedBeaconBlock): bool
 
 proc get_v1_validator_attestation_data(slot: Slot, committee_index: CommitteeIndex): AttestationData
@@ -31,16 +24,17 @@ proc get_v1_validator_attestation_data(slot: Slot, committee_index: CommitteeInd
 # https://docs.google.com/spreadsheets/d/1kVIx6GvzVLwNYbcd-Fj8YUlPf4qGrWUlS35uaTnIAVg/edit?disco=AAAAGh7r_fQ
 proc get_v1_validator_aggregate_attestation(attestation_data: AttestationData): Attestation
 
-# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
 proc post_v1_validator_aggregate_and_proof(payload: SignedAggregateAndProof): bool
 
 # this is a POST instead of a GET because of this: https://docs.google.com/spreadsheets/d/1kVIx6GvzVLwNYbcd-Fj8YUlPf4qGrWUlS35uaTnIAVg/edit?disco=AAAAJk5rbKA
+# TODO epoch is part of the REST path
 proc post_v1_validator_duties_attester(epoch: Epoch, public_keys: seq[ValidatorPubKey]): seq[AttesterDuties]
 
+# TODO epoch is part of the REST path
 proc get_v1_validator_duties_proposer(epoch: Epoch): seq[ValidatorPubkeySlotPair]
 
-proc post_v1_validator_beacon_committee_subscription(committee_index: CommitteeIndex,
-                                                     slot: Slot,
-                                                     aggregator: bool,
-                                                     validator_pubkey: ValidatorPubKey,
-                                                     slot_signature: ValidatorSig)
+proc post_v1_validator_beacon_committee_subscriptions(committee_index: CommitteeIndex,
+                                                      slot: Slot,
+                                                      aggregator: bool,
+                                                      validator_pubkey: ValidatorPubKey,
+                                                      slot_signature: ValidatorSig): bool

beacon_chain/spec/eth2_apis/validator_callsigs_types.nim

@@ -1,27 +0,0 @@
-import
-  # Standard library
-  options,
-  # Local modules
-  # TODO for some reason "../[datatypes, digest, crypto]" results in "Error: cannot open file"
-  ../datatypes,
-  ../digest,
-  ../crypto
-
-type
-  AttesterDuties* = object
-    public_key*: ValidatorPubKey
-    committee_index*: CommitteeIndex
-    committee_length*: uint64
-    validator_committee_index*: uint64
-    slot*: Slot
-
-  # TODO do we even need this? how about a simple tuple (alias)?
-  ValidatorPubkeySlotPair* = object
-    public_key*: ValidatorPubKey
-    slot*: Slot
-
-  # TODO do we even need this? how about a simple tuple (alias)?
-  BeaconGenesisTuple* = object
-    genesis_time*: uint64
-    genesis_validators_root*: Eth2Digest
-    genesis_fork_version*: Version

beacon_chain/validator_api.nim

@@ -7,7 +7,7 @@
 
 import
   # Standard library
-  tables, strutils,
+  tables, strutils, parseutils,
 
   # Nimble packages
   stew/[objects],
@@ -19,7 +19,7 @@ import
   block_pool, ssz/merkleization,
   beacon_node_common, beacon_node_types,
   validator_duties, eth2_network,
-  spec/eth2_apis/validator_callsigs_types,
+  spec/eth2_apis/callsigs_types,
   eth2_json_rpc_serialization
 
 type
@@ -27,64 +27,102 @@ type
 
 logScope: topics = "valapi"
 
+# TODO Probably the `beacon` ones should be defined elsewhere...?
+
 proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
 
-  # TODO Probably the `beacon` ones (and not `validator`) should be defined elsewhere...
-  rpcServer.rpc("get_v1_beacon_states_fork") do (stateId: string) -> Fork:
-    notice "== get_v1_beacon_states_fork", stateId = stateId
+  template withStateForSlot(stateId: string, body: untyped): untyped =
+    var res: BiggestInt
+    if parseBiggestInt(stateId, res) == stateId.len:
+      raise newException(CatchableError, "Not a valid slot number")
+    let head = node.updateHead()
+    let blockSlot = head.atSlot(res.Slot)
+    node.blockPool.withState(node.blockPool.tmpState, blockSlot):
+      body
+
+  rpcServer.rpc("get_v1_beacon_genesis") do () -> BeaconGenesisTuple:
+    debug "get_v1_beacon_genesis"
+    return (genesis_time: node.blockPool.headState.data.data.genesis_time,
+             genesis_validators_root:
+              node.blockPool.headState.data.data.genesis_validators_root,
+             genesis_fork_version: Version(GENESIS_FORK_VERSION))
+
+  rpcServer.rpc("get_v1_beacon_states_root") do (stateId: string) -> Eth2Digest:
+    debug "get_v1_beacon_states_root", stateId = stateId
+    # TODO do we need to call node.updateHead() before using headState?
+    result = case stateId:
+      of "head":
+        node.blockPool.headState.blck.root
+      of "genesis":
+        node.blockPool.headState.data.data.genesis_validators_root
+      of "finalized":
+        node.blockPool.headState.data.data.finalized_checkpoint.root
+      of "justified":
+        node.blockPool.headState.data.data.current_justified_checkpoint.root
+      else:
+        if stateId.startsWith("0x"):
+          # TODO not sure if `fromHex` is the right thing here...
+          # https://github.com/ethereum/eth2.0-APIs/issues/37#issuecomment-638566144
+          # we return whatever was passed to us (this is a nonsense request)
+          fromHex(Eth2Digest, stateId[2..<stateId.len]) # skip first 2 chars
+        else:
+          withStateForSlot(stateId):
+            hashedState.root
+
+  rpcServer.rpc("get_v1_beacon_states_fork") do (stateId: string) -> Fork:
+    debug "get_v1_beacon_states_fork", stateId = stateId
     result = case stateId:
       of "head":
-        discard node.updateHead() # TODO do we need this?
         node.blockPool.headState.data.data.fork
       of "genesis":
         Fork(previous_version: Version(GENESIS_FORK_VERSION),
              current_version: Version(GENESIS_FORK_VERSION),
              epoch: GENESIS_EPOCH)
       of "finalized":
-        # TODO
-        Fork()
+        node.blockPool.withState(node.blockPool.tmpState, node.blockPool.finalizedHead):
+          state.fork
       of "justified":
-        # TODO
-        Fork()
+        node.blockPool.justifiedState.data.data.fork
       else:
-        # TODO parse `stateId` as either a number (slot) or a hash (stateRoot)
-        Fork()
+        if stateId.startsWith("0x"):
+          # TODO not sure if `fromHex` is the right thing here...
+          # https://github.com/ethereum/eth2.0-APIs/issues/37#issuecomment-638566144
+          let blckRoot = fromHex(Eth2Digest, stateId[2..<stateId.len]) # skip first 2 chars
+          let blckRef = node.blockPool.getRef(blckRoot)
+          if blckRef.isNil:
+            raise newException(CatchableError, "Block not found")
+          let blckSlot = blckRef.atSlot(blckRef.slot)
+          node.blockPool.withState(node.blockPool.tmpState, blckSlot):
+            state.fork
+        else:
+          withStateForSlot(stateId):
+            state.fork
 
-  # TODO Probably the `beacon` ones (and not `validator`) should be defined elsewhere...
-  rpcServer.rpc("get_v1_beacon_genesis") do () -> BeaconGenesisTuple:
-    notice "== get_v1_beacon_genesis"
-    return BeaconGenesisTuple(genesis_time: node.blockPool.headState.data.data.genesis_time,
-                              genesis_validators_root: node.blockPool.headState.data.data.genesis_validators_root,
-                              genesis_fork_version: Version(GENESIS_FORK_VERSION))
-
-  rpcServer.rpc("post_v1_beacon_pool_attestations") do (attestation: Attestation) -> bool:
-    #notice "== post_v1_beacon_pool_attestations"
+  rpcServer.rpc("post_v1_beacon_pool_attestations") do (
+      attestation: Attestation) -> bool:
     node.sendAttestation(attestation)
     return true
 
   rpcServer.rpc("get_v1_validator_blocks") do (
       slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig) -> BeaconBlock:
-    notice "== get_v1_validator_blocks", slot = slot
+    debug "get_v1_validator_blocks", slot = slot
     let head = node.updateHead()
-
     let proposer = node.blockPool.getProposer(head, slot)
-    # TODO how do we handle the case when we cannot return a meaningful block? 404...
-    doAssert(proposer.isSome())
-
+    if proposer.isNone():
+      raise newException(CatchableError, "could not retrieve block for slot: " & $slot)
     let valInfo = ValidatorInfoForMakeBeaconBlock(kind: viRandao_reveal,
                                                   randao_reveal: randao_reveal)
     let res = makeBeaconBlockForHeadAndSlot(
       node, valInfo, proposer.get()[0], graffiti, head, slot)
-
-    # TODO how do we handle the case when we cannot return a meaningful block? 404...
-    doAssert(res.message.isSome())
-    return res.message.get(BeaconBlock()) # returning a default if empty
+    if res.message.isNone():
+      raise newException(CatchableError, "could not retrieve block for slot: " & $slot)
+    return res.message.get()
 
   rpcServer.rpc("post_v1_beacon_blocks") do (body: SignedBeaconBlock) -> bool:
-    notice "== post_v1_beacon_blocks"
+    debug "post_v1_beacon_blocks",
+      slot = body.message.slot,
+      prop_idx = body.message.proposer_index
 
-    logScope: pcs = "block_proposal"
-    
     let head = node.updateHead()
     if head.slot >= body.message.slot:
       warn "Skipping proposal, have newer head already",
@@ -92,14 +130,15 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
         headBlockRoot = shortLog(head.root),
         slot = shortLog(body.message.slot),
         cat = "fastforward"
-      return false
-    return head != await proposeSignedBlock(node, head, AttachedValidator(), 
-                                            body, hash_tree_root(body.message))
+      raise newException(CatchableError,
+        "Proposal is for a past slot: " & $body.message.slot)
+    if head == await proposeSignedBlock(node, head, AttachedValidator(), 
+                                        body, hash_tree_root(body.message)):
+      raise newException(CatchableError, "Could not propose block")
+    return true
 
   rpcServer.rpc("get_v1_validator_attestation_data") do (
       slot: Slot, committee_index: CommitteeIndex) -> AttestationData:
-    #notice "== get_v1_validator_attestation_data"
-    # Obtain the data to form an attestation
     let head = node.updateHead()
     let attestationHead = head.atSlot(slot)
     node.blockPool.withState(node.blockPool.tmpState, attestationHead):
@@ -107,45 +146,43 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
 
   rpcServer.rpc("get_v1_validator_aggregate_attestation") do (
       attestation_data: AttestationData)-> Attestation:
-    notice "== get_v1_validator_aggregate_attestation"
+    debug "get_v1_validator_aggregate_attestation"
 
   rpcServer.rpc("post_v1_validator_aggregate_and_proof") do (
       payload: SignedAggregateAndProof) -> bool:
-    notice "== post_v1_validator_aggregate_and_proof"
-    # TODO is this enough?
     node.network.broadcast(node.topicAggregateAndProofs, payload)
     return true
 
   rpcServer.rpc("post_v1_validator_duties_attester") do (
       epoch: Epoch, public_keys: seq[ValidatorPubKey]) -> seq[AttesterDuties]:
-    notice "== post_v1_validator_duties_attester", epoch = epoch
-    discard node.updateHead() # TODO do we need this?
-    for pubkey in public_keys:
-      let idx = node.blockPool.headState.data.data.validators.asSeq.findIt(it.pubKey == pubkey)
-      if idx != -1:
-        # TODO this might crash if the requested epoch is further than the BN epoch
-        # because of this: `doAssert epoch <= next_epoch`
-        let res = node.blockPool.headState.data.data.get_committee_assignment(
-          epoch, idx.ValidatorIndex)
-        if res.isSome:
-          result.add(AttesterDuties(public_key: pubkey,
-                                    committee_index: res.get.b,
-                                    committee_length: res.get.a.len.uint64,
-                                    validator_committee_index: res.get.a.find(idx.ValidatorIndex).uint64,
-                                    slot: res.get.c))
+    debug "post_v1_validator_duties_attester", epoch = epoch
+    let head = node.updateHead()
+    let attestationHead = head.atSlot(compute_start_slot_at_epoch(epoch))
+    node.blockPool.withState(node.blockPool.tmpState, attestationHead):
+      for pubkey in public_keys:
+        let idx = state.validators.asSeq.findIt(it.pubKey == pubkey)
+        if idx == -1:
+          continue
+        let ca = state.get_committee_assignment(epoch, idx.ValidatorIndex)
+        if ca.isSome:
+          result.add((public_key: pubkey,
+                      committee_index: ca.get.b,
+                      committee_length: ca.get.a.len.uint64,
+                      validator_committee_index: ca.get.a.find(idx.ValidatorIndex).uint64,
+                      slot: ca.get.c))
 
   rpcServer.rpc("get_v1_validator_duties_proposer") do (
       epoch: Epoch) -> seq[ValidatorPubkeySlotPair]:
-    notice "== get_v1_validator_duties_proposer", epoch = epoch
+    debug "get_v1_validator_duties_proposer", epoch = epoch
     let head = node.updateHead()
     for i in 0 ..< SLOTS_PER_EPOCH:
       let currSlot = (compute_start_slot_at_epoch(epoch).int + i).Slot
       let proposer = node.blockPool.getProposer(head, currSlot)
       if proposer.isSome():
-        result.add(ValidatorPubkeySlotPair(public_key: proposer.get()[1], slot: currSlot))
+        result.add((public_key: proposer.get()[1], slot: currSlot))
 
-  rpcServer.rpc("post_v1_validator_beacon_committee_subscription") do (
+  rpcServer.rpc("post_v1_validator_beacon_committee_subscriptions") do (
       committee_index: CommitteeIndex, slot: Slot, aggregator: bool,
-      validator_pubkey: ValidatorPubKey, slot_signature: ValidatorSig):
-    notice "== post_v1_validator_beacon_committee_subscription"
-    # TODO
+      validator_pubkey: ValidatorPubKey, slot_signature: ValidatorSig) -> bool:
+    debug "post_v1_validator_beacon_committee_subscriptions"
+    raise newException(CatchableError, "Not implemented")

beacon_chain/validator_client.nim

@@ -22,7 +22,7 @@ import
   nimbus_binary_common,
   version, ssz/merkleization,
   sync_manager, keystore_management,
-  spec/eth2_apis/validator_callsigs_types,
+  spec/eth2_apis/callsigs_types,
   eth2_json_rpc_serialization
 
 logScope: topics = "vc"
@@ -31,6 +31,7 @@ template sourceDir: string = currentSourcePath.rsplit(DirSep, 1)[0]
 
 ## Generate client convenience marshalling wrappers from forward declarations
 createRpcSigs(RpcClient, sourceDir / "spec" / "eth2_apis" / "validator_callsigs.nim")
+createRpcSigs(RpcClient, sourceDir / "spec" / "eth2_apis" / "beacon_callsigs.nim")
 
 type
   ValidatorClient = ref object
@@ -39,31 +40,66 @@ type
     beaconClock: BeaconClock
     attachedValidators: ValidatorPool
     fork: Fork
-    proposalsForEpoch: Table[Slot, ValidatorPubKey]
-    attestationsForEpoch: Table[Slot, seq[AttesterDuties]]
+    proposalsForCurrentEpoch: Table[Slot, ValidatorPubKey]
+    attestationsForEpoch: Table[Epoch, Table[Slot, seq[AttesterDuties]]]
     beaconGenesis: BeaconGenesisTuple
 
+proc connectToBN(vc: ValidatorClient) {.gcsafe, async.} =
+  while true:
+    try:
+      await vc.client.connect($vc.config.rpcAddress, Port(vc.config.rpcPort))
+      info "Connected to BN",
+        port = vc.config.rpcPort,
+        address = vc.config.rpcAddress
+      return
+    except CatchableError as err:
+      warn "Could not connect to the BN - retrying!", err = err.msg
+    await sleepAsync(chronos.seconds(1)) # 1 second before retrying
+
+template attemptUntilSuccess(vc: ValidatorClient, body: untyped) =
+  while true:
+    try:
+      body
+      break
+    except CatchableError as err:
+      warn "Caught an unexpected error", err = err.msg
+    waitFor vc.connectToBN()
+
 proc getValidatorDutiesForEpoch(vc: ValidatorClient, epoch: Epoch) {.gcsafe, async.} =
   let proposals = await vc.client.get_v1_validator_duties_proposer(epoch)
   # update the block proposal duties this VC should do during this epoch
-  vc.proposalsForEpoch.clear()
+  vc.proposalsForCurrentEpoch.clear()
   for curr in proposals:
     if vc.attachedValidators.validators.contains curr.public_key:
-      vc.proposalsForEpoch.add(curr.slot, curr.public_key)
+      vc.proposalsForCurrentEpoch.add(curr.slot, curr.public_key)
 
   # couldn't use mapIt in ANY shape or form so reverting to raw loops - sorry Sean Parent :|
   var validatorPubkeys: seq[ValidatorPubKey]
   for key in vc.attachedValidators.validators.keys:
     validatorPubkeys.add key
-  # update the attestation duties this VC should do during this epoch
-  let attestations = await vc.client.post_v1_validator_duties_attester(
-    epoch, validatorPubkeys)
-  vc.attestationsForEpoch.clear()
-  for a in attestations:
-    if vc.attestationsForEpoch.hasKeyOrPut(a.slot, @[a]):
-      vc.attestationsForEpoch[a.slot].add(a)
+
+  proc getAttesterDutiesForEpoch(epoch: Epoch) {.gcsafe, async.} =
+    let attestations = await vc.client.post_v1_validator_duties_attester(
+      epoch, validatorPubkeys)
+    # make sure there's an entry
+    if not vc.attestationsForEpoch.contains epoch:
+      vc.attestationsForEpoch.add(epoch, Table[Slot, seq[AttesterDuties]]())
+    for a in attestations:
+      if vc.attestationsForEpoch[epoch].hasKeyOrPut(a.slot, @[a]):
+        vc.attestationsForEpoch[epoch][a.slot].add(a)
+
+  # obtain the attestation duties this VC should do during the next epoch
+  await getAttesterDutiesForEpoch(epoch + 1)
+  # also get the attestation duties for the current epoch if missing
+  if not vc.attestationsForEpoch.contains epoch:
+    await getAttesterDutiesForEpoch(epoch)
+  # cleanup old epoch attestation duties
+  vc.attestationsForEpoch.del(epoch - 1)
+  # TODO handle subscriptions to beacon committees for both the next epoch and
+  # for the current if missing (beacon_committee_subscriptions from the REST api)
 
   # for now we will get the fork each time we update the validator duties for each epoch
+  # TODO should poll occasionally `/v1/config/fork_schedule`
   vc.fork = await vc.client.get_v1_beacon_states_fork("head")
 
 proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, async.} =
@@ -76,6 +112,7 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
   let
     slot = wallSlot.slot # afterGenesis == true!
     nextSlot = slot + 1
+    epoch = slot.compute_epoch_at_slot
 
   info "Slot start",
     lastSlot = shortLog(lastSlot),
@@ -91,11 +128,11 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
     # could take up time for attesting... Perhaps this should be called more
     # than once per epoch because of forks & other events...
     if slot.isEpoch:
-      await getValidatorDutiesForEpoch(vc, slot.compute_epoch_at_slot)
+      await getValidatorDutiesForEpoch(vc, epoch)
 
     # check if we have a validator which needs to propose on this slot
-    if vc.proposalsForEpoch.contains slot:
-      let public_key = vc.proposalsForEpoch[slot]
+    if vc.proposalsForCurrentEpoch.contains slot:
+      let public_key = vc.proposalsForCurrentEpoch[slot]
       let validator = vc.attachedValidators.validators[public_key]
 
       let randao_reveal = validator.genRandaoReveal(
@@ -121,8 +158,8 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
       seconds(int64(SECONDS_PER_SLOT)) div 3, slot, "Waiting to send attestations")
 
     # check if we have validators which need to attest on this slot
-    if vc.attestationsForEpoch.contains slot:
-      for a in vc.attestationsForEpoch[slot]:
+    if vc.attestationsForEpoch[epoch].contains slot:
+      for a in vc.attestationsForEpoch[epoch][slot]:
         let validator = vc.attachedValidators.validators[a.public_key]
 
         let ad = await vc.client.get_v1_validator_attestation_data(slot, a.committee_index)
@@ -135,7 +172,8 @@ proc onSlotStart(vc: ValidatorClient, lastSlot, scheduledSlot: Slot) {.gcsafe, a
         discard await vc.client.post_v1_beacon_pool_attestations(attestation)
 
   except CatchableError as err:
-    error "Caught an unexpected error", err = err.msg
+    warn "Caught an unexpected error", err = err.msg, slot = shortLog(slot)
+    await vc.connectToBN()
 
   let
     nextSlotStart = saturate(vc.beaconClock.fromNow(nextSlot))
@@ -177,34 +215,27 @@ programMain:
 
     var vc = ValidatorClient(
       config: config,
-      client: newRpcHttpClient(),
-      attachedValidators: ValidatorPool.init()
+      client: newRpcHttpClient()
     )
-    vc.proposalsForEpoch.init()
-    vc.attestationsForEpoch.init()
 
     # load all the validators from the data dir into memory
     for curr in vc.config.validatorKeys:
       vc.attachedValidators.addLocalValidator(curr.toPubKey, curr)
 
-    # TODO perhaps we should handle the case if the BN is down and try to connect to it
-    # untill success, and also later on disconnets we should continue trying to reconnect
-    waitFor vc.client.connect("localhost", Port(config.rpcPort)) # TODO: use config.rpcAddress
-    info "Connected to beacon node", port = config.rpcPort
+    waitFor vc.connectToBN()
 
-    # init the beacon clock
-    vc.beaconGenesis = waitFor vc.client.get_v1_beacon_genesis()
-    vc.beaconClock = BeaconClock.init(vc.beaconGenesis.genesis_time)
+    vc.attemptUntilSuccess:
+      # init the beacon clock
+      vc.beaconGenesis = waitFor vc.client.get_v1_beacon_genesis()
+      vc.beaconClock = BeaconClock.init(vc.beaconGenesis.genesis_time)
 
     let
       curSlot = vc.beaconClock.now().slotOrZero()
       nextSlot = curSlot + 1 # No earlier than GENESIS_SLOT + 1
       fromNow = saturate(vc.beaconClock.fromNow(nextSlot))
 
-    # onSlotStart() requests the validator duties only on the start of each epoch
-    # so we should request the duties here when the VC binary boots up in order
-    # to handle the case when in the middle of an epoch. Also for the genesis slot.
-    waitFor vc.getValidatorDutiesForEpoch(curSlot.compute_epoch_at_slot)
+    vc.attemptUntilSuccess:
+      waitFor vc.getValidatorDutiesForEpoch(curSlot.compute_epoch_at_slot)
 
     info "Scheduling first slot action",
       beaconTime = shortLog(vc.beaconClock.now()),

tests/simulation/run_node.sh

@@ -53,7 +53,7 @@ VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
 if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
   # if using validator client binaries in addition to beacon nodes
   # we will split the keys for this instance in half between the BN and the VC
-  if [ "${SPLIT_VALIDATORS_BETWEEN_BN_AND_VC:-}" == "yes" ]; then
+  if [ "${BN_VC_VALIDATOR_SPLIT:-}" == "yes" ]; then
     ATTACHED_VALIDATORS=$((VALIDATORS_PER_NODE / 2))
   else
     ATTACHED_VALIDATORS=$VALIDATORS_PER_NODE

tests/simulation/run_validator.sh

@@ -15,26 +15,34 @@ source "${SIM_ROOT}/../../env.sh"
 
 cd "$GIT_ROOT"
 
-VC_DATA_DIR="${SIMULATION_DIR}/validator-$NODE_ID"
+NODE_DATA_DIR="${SIMULATION_DIR}/validator-$NODE_ID"
+NODE_VALIDATORS_DIR=$NODE_DATA_DIR/validators/
+NODE_SECRETS_DIR=$NODE_DATA_DIR/secrets/
 
-mkdir -p "$VC_DATA_DIR/validators"
-rm -f $VC_DATA_DIR/validators/*
+rm -rf "$NODE_VALIDATORS_DIR"
+mkdir -p "$NODE_VALIDATORS_DIR"
+
+rm -rf "$NODE_SECRETS_DIR"
+mkdir -p "$NODE_SECRETS_DIR"
+
+VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
 
 if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
   # we will split the keys for this instance in half between the BN and the VC
-  VALIDATORS_PER_NODE=$((NUM_VALIDATORS / TOTAL_NODES))
-  VALIDATORS_PER_NODE_HALF=$((VALIDATORS_PER_NODE / 2))
-  FIRST_VALIDATOR_IDX=$(( VALIDATORS_PER_NODE * NODE_ID + VALIDATORS_PER_NODE_HALF))
-  LAST_VALIDATOR_IDX=$(( FIRST_VALIDATOR_IDX + VALIDATORS_PER_NODE_HALF - 1 ))
+  ATTACHED_VALIDATORS=$((VALIDATORS_PER_NODE / 2))
 
   pushd "$VALIDATORS_DIR" >/dev/null
-    cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) "$VC_DATA_DIR/validators"
+  for VALIDATOR in $(ls | tail -n +$(( ($VALIDATORS_PER_NODE * $NODE_ID) + 1 + $ATTACHED_VALIDATORS )) | head -n $ATTACHED_VALIDATORS); do
+      cp -ar "$VALIDATOR" "$NODE_VALIDATORS_DIR"
+      cp -a "$SECRETS_DIR/$VALIDATOR" "$NODE_SECRETS_DIR"
+    done
   popd >/dev/null
 fi
 
-cd "$VC_DATA_DIR"
+cd "$NODE_DATA_DIR"
 
 $VALIDATOR_CLIENT_BIN \
   --log-level=${LOG_LEVEL:-DEBUG} \
-  --data-dir=$VC_DATA_DIR \
+  --data-dir=$NODE_DATA_DIR \
+  --secrets-dir=$NODE_SECRETS_DIR \
   --rpc-port="$(( $BASE_RPC_PORT + $NODE_ID ))"

tests/simulation/start.sh

@@ -179,6 +179,7 @@ LAST_WAITING_NODE=0
 function run_cmd {
   i=$1
   CMD=$2
+  bin_name=$3
   if [[ "$USE_TMUX" != "no" ]]; then
     echo "Starting node $i..."
     echo $TMUX split-window -t "${TMUX_SESSION_NAME}" "$CMD"
@@ -191,7 +192,7 @@ function run_cmd {
       SLEEP="3"
     fi
     # "multitail" closes the corresponding panel when a command exits, so let's make sure it doesn't exit
-    COMMANDS+=( " -cT ansi -t 'node #$i' -l 'sleep $SLEEP; $CMD; echo [node execution completed]; while true; do sleep 100; done'" )
+    COMMANDS+=( " -cT ansi -t '$bin_name #$i' -l 'sleep $SLEEP; $CMD; echo [node execution completed]; while true; do sleep 100; done'" )
   else
     eval "${CMD}" &
   fi
@@ -209,11 +210,11 @@ for i in $(seq $MASTER_NODE -1 $TOTAL_USER_NODES); do
     done
   fi
 
-  run_cmd $i "${SIM_ROOT}/run_node.sh ${i} --verify-finalization"
+  run_cmd $i "${SIM_ROOT}/run_node.sh ${i} --verify-finalization" "node"
 
-  if [ "${SPLIT_VALIDATORS_BETWEEN_BN_AND_VC:-}" == "yes" ]; then
+  if [ "${BN_VC_VALIDATOR_SPLIT:-}" == "yes" ]; then
     # start the VC with a few seconds of delay so that we can connect through RPC
-    run_cmd $i "sleep 3 && ${SIM_ROOT}/run_validator.sh ${i}"
+    run_cmd $i "sleep 3 && ${SIM_ROOT}/run_validator.sh ${i}" "validator"
   fi
 done
 

tests/simulation/vars.sh

@@ -46,6 +46,3 @@ else
   WEB3_ARG=""
   DEPOSIT_CONTRACT_ADDRESS="0x"
 fi
-
-# uncomment to enable the use of VCs in addition to BNs - will split the validators equally
-#SPLIT_VALIDATORS_BETWEEN_BN_AND_VC="yes"