Merge pull request #21 from status-im/zah-skeleton-wip

Beacon node skeleton

README.md

@@ -21,7 +21,7 @@ Ethereum Foundation uses:
 Nim NEP-1 recommends:
   - camelCase for fields and procedure names
   - PascalCase for constants
-  - PsacalCase for types
+  - PascalCase for types
 
 To facilitate collaboration and comparison, Nim-beacon-chain uses the Ethereum Foundation convention.
 

beacon_chain.nimble

@@ -3,13 +3,22 @@ version       = "0.0.1"
 author        = "Status Research & Development GmbH"
 description   = "Eth2.0 research implementation of the beacon chain"
 license       = "MIT or Apache License 2.0"
-srcDir        = "src"
+installDirs   = @["beacon_chain"]
+bin           = @["beacon_chain/beacon_node"]
 
 ### Dependencies
 requires "nim >= 0.18.0",
   "eth_common",
+  "eth_keys",
   "nimcrypto",
-  "https://github.com/status-im/nim-milagro-crypto#master"
+  "https://github.com/status-im/nim-milagro-crypto#master",
+  "eth_p2p",
+  "ranges",
+  "chronicles",
+  "confutils",
+  "serialization",
+  "json_serialization",
+  "json_rpc"
 
 ### Helper functions
 proc test(name: string, defaultLang = "c") =

beacon_chain/beacon_chain_db.nim

@@ -0,0 +1,28 @@
+import
+  os, json,
+  chronicles, json_serialization, eth_common/eth_types_json_serialization,
+  spec/datatypes
+
+type
+  BeaconChainDB* = ref object
+    dataRoot: string
+
+  BeaconStateRef* = ref BeaconState
+
+proc init*(T: type BeaconChainDB, dataDir: string): BeaconChainDB =
+  new result
+  result.dataRoot = dataDir / "beacon_db"
+  createDir(result.dataRoot)
+
+proc lastFinalizedState*(db: BeaconChainDB): BeaconStateRef =
+  try:
+    var stateJson = parseJson readFile(db.dataRoot / "BeaconState.json")
+    # TODO implement this
+  except:
+    return nil
+
+proc persistBlock*(db: BeaconChainDB, s: BeaconState, b: BeaconBlock) =
+  let stateJson = StringJsonWriter.encode(s, pretty = true)
+  writeFile(db.dataRoot / "BeaconState.json", stateJson)
+  debug "State persisted"
+

beacon_chain/beacon_node.nim

@@ -0,0 +1,195 @@
+import
+  os, net,
+  asyncdispatch2, chronicles, confutils, eth_p2p, eth_keys,
+  spec/[beaconstate, datatypes], conf, time, fork_choice,
+  beacon_chain_db, validator_pool, mainchain_monitor,
+  sync_protocol, gossipsub_protocol, trusted_state_snapshots
+
+type
+  BeaconNode* = ref object
+    beaconState*: BeaconState
+    network*: EthereumNode
+    db*: BeaconChainDB
+    config*: BeaconNodeConf
+    keys*: KeyPair
+    attachedValidators: ValidatorPool
+    attestations: AttestationPool
+    headBlock: BeaconBlock
+    mainchainMonitor: MainchainMonitor
+
+const
+  version = "v0.1" # TODO: read this from the nimble file
+  clientId = "nimbus beacon node " & version
+
+  topicBeaconBlocks = "ethereum/2.1/beacon_chain/blocks"
+  topicAttestations = "ethereum/2.1/beacon_chain/attestations"
+
+proc ensureNetworkKeys*(dataDir: string): KeyPair =
+  # TODO:
+  # 1. Check if keys already exist in the data dir
+  # 2. Generate new ones and save them in the directory
+  # if necessary
+  return newKeyPair()
+
+proc init*(T: type BeaconNode, conf: BeaconNodeConf): T =
+  new result
+  result.config = conf
+  result.db = BeaconChainDB.init(string conf.dataDir)
+  result.keys = ensureNetworkKeys(string conf.dataDir)
+
+  var address: Address
+  address.ip = parseIpAddress("0.0.0.0")
+  address.tcpPort = Port(conf.tcpPort)
+  address.udpPort = Port(conf.udpPort)
+  result.network = newEthereumNode(result.keys, address, 0, nil, clientId)
+
+proc sync*(node: BeaconNode): Future[bool] {.async.} =
+  let persistedState = node.db.lastFinalizedState()
+  if persistedState.isNil or
+     persistedState[].slotDistanceFromNow() > WEAK_SUBJECTVITY_PERIOD:
+    node.beaconState = await obtainTrustedStateSnapshot(node.db)
+  else:
+    node.beaconState = persistedState[]
+    var targetSlot = toSlot timeSinceGenesis(node.beaconState)
+
+    while node.beaconState.last_finalized_slot.int < targetSlot:
+      var (peer, changeLog) = await node.network.getValidatorChangeLog(
+        node.beaconState.validator_set_delta_hash_chain)
+
+      if peer == nil:
+        error "Failed to sync with any peer"
+        return false
+
+      if applyValidatorChangeLog(changeLog, node.beaconState):
+        node.db.persistBlock(node.beaconState, changeLog.signedBlock)
+      else:
+        warn "Ignoring invalid validator change log", sentFrom = peer
+
+  return true
+
+proc addLocalValidators*(node: BeaconNode) =
+  for validator in node.config.validatorKeys:
+    # TODO:
+    # 1. Parse the validator keys
+    #
+    # 2. Check whether the validators exist in the beacon state.
+    #    (Report a warning otherwise)
+    #
+    # 3. Add the validators to node.attachedValidators
+    discard
+
+proc getAttachedValidator(node: BeaconNode, idx: int): AttachedValidator =
+  let validatorKey = node.beaconState.validators[idx].pubkey
+  return node.attachedValidators.getValidator(validatorKey)
+
+proc makeAttestation(node: BeaconNode,
+                     validator: AttachedValidator) {.async.} =
+  var attestation: Attestation
+  attestation.validator = validator.idx
+
+  # TODO: Populate attestation.data
+
+  attestation.signature = await validator.signAttestation(attestation.data)
+  await node.network.broadcast(topicAttestations, attestation)
+
+proc proposeBlock(node: BeaconNode,
+                  validator: AttachedValidator,
+                  slot: int) {.async.} =
+  var proposal: BeaconBlock
+
+  # TODO:
+  # 1. Produce a RANDAO reveal from attachedVadalidator.randaoSecret
+  # and its matching ValidatorRecord.
+
+  # 2. Get ancestors from the beacon_db
+
+  # 3. Calculate the correct state hash
+
+  proposal.candidate_pow_receipt_root =
+    node.mainchainMonitor.getBeaconBlockRef()
+
+  for a in node.attestations.each(firstSlot = node.headBlock.slot.int + 1,
+                                  lastSlot = slot - MIN_ATTESTATION_INCLUSION_DELAY):
+    # TODO: this is not quite right,
+    # the attestations from individual validators have to be merged.
+    # proposal.attestations.add a
+    discard
+
+  for r in node.mainchainMonitor.getValidatorActions(
+      node.headBlock.candidate_pow_receipt_root,
+      proposal.candidate_pow_receipt_root):
+    proposal.specials.add r
+
+  var signedData: ProposalSignedData
+  # TODO: populate the signed data
+
+  proposal.proposer_signature = await validator.signBlockProposal(signedData)
+  await node.network.broadcast(topicBeaconBlocks, proposal)
+
+proc scheduleCycleActions(node: BeaconNode) =
+  ## This schedules the required block proposals and
+  ## attestations from our attached validators.
+  let cycleStart = node.beaconState.last_state_recalculation_slot.int
+
+  for i in 0 ..< CYCLE_LENGTH:
+    # Schedule block proposals
+    let
+      slot = cycleStart + i
+      proposerIdx = get_beacon_proposer_idx(node.beaconState, slot)
+      attachedValidator = node.getAttachedValidator(proposerIdx)
+
+    if attachedValidator != nil:
+      # TODO:
+      # Warm-up the proposer earlier to try to obtain previous
+      # missing blocks if necessary
+
+      addTimer(node.beaconState.slotStart(slot)) do (p: pointer):
+        asyncCheck proposeBlock(node, attachedValidator, slot)
+
+    # Schedule attestations
+    let
+      committeesIdx = get_shard_and_committees_idx(node.beaconState, slot)
+
+    for shard in node.beaconState.shard_and_committee_for_slots[committees_idx]:
+      for validatorIdx in shard.committee:
+        let attachedValidator = node.getAttachedValidator(validatorIdx)
+        if attachedValidator != nil:
+          addTimer(node.beaconState.slotMiddle(slot)) do (p: pointer):
+            asyncCheck makeAttestation(node, attachedValidator)
+
+proc processBlocks*(node: BeaconNode) {.async.} =
+  node.scheduleCycleActions()
+
+  node.network.subscribe(topicBeaconBlocks) do (b: BeaconBlock):
+    # TODO:
+    #
+    # 1. Check for missing blocks and obtain them
+    #
+    # 2. Apply fork-choice rule (update node.headBlock)
+    #
+    # 3. Peform block processing / state recalculation / etc
+    #
+
+    if b.slot mod CYCLE_LENGTH == 0:
+      node.scheduleCycleActions()
+      node.attestations.discardHistoryToSlot(b.slot)
+
+  node.network.subscribe(topicAttestations) do (a: Attestation):
+    # TODO
+    #
+    # 1. Validate the attestation
+
+    node.attestations.add(a, node.beaconState)
+
+when isMainModule:
+  let config = BeaconNodeConf.load()
+  waitFor syncrhronizeClock()
+  var node = BeaconNode.init config
+
+  if not waitFor node.sync():
+    quit 1
+
+  node.addLocalValidators()
+
+  waitFor node.processBlocks()
+

beacon_chain/conf.nim

@@ -0,0 +1,33 @@
+import
+  confutils/defs
+
+type
+  ValidatorKeyPath* = distinct string
+
+  BeaconNodeConf* = object
+    dataDir* {.
+      desc: "The directory where nimbus will store all blockchain data.",
+      shorthand: "d",
+      defaultValue: getConfigDir() / "nimbus".}: DirPath
+
+    bootstrapNodes* {.
+      desc: "Specifies one or more bootstrap nodes to use when connecting to the network.",
+      shorthand: "b".}: seq[string]
+
+    tcpPort* {.
+      desc: "TCP listening port".}: int
+
+    udpPort* {.
+      desc: "UDP listening port".}: int
+
+    validatorKeys* {.
+      desc: "A path to a pair of public and private keys for a validator. " &
+            "Nimbus will automatically add the extensions .privkey and .pubkey.",
+      shorthand: "v".}: seq[ValidatorKeyPath]
+
+proc parse*(T: type ValidatorKeyPath, input: TaintedString): T =
+  # TODO:
+  # Check that the entered string is a valid base file name and
+  # that it has matching .privkey, .pubkey and .randaosecret files
+  T(input)
+

beacon_chain/fork_choice.nim

@@ -0,0 +1,67 @@
+import
+  deques,
+  spec/[datatypes, crypto]
+
+type
+  Attestation* = object
+    validator*: int
+    data*: AttestationSignedData
+    signature*: ValidatorSig
+
+  AttestationPool* = object
+    attestations: Deque[seq[Attestation]]
+    startingSlot: int
+
+proc init*(T: type AttestationPool, startingSlot: int): T =
+  result.attestationsPerSlot = initDeque[seq[Attestation]]()
+  result.startingSlot = startingSlot
+
+proc setLen*[T](d: var Deque[T], len: int) =
+  # TODO: The upstream `Deque` type should gain a proper resize API
+  let delta = len - d.len
+  if delta > 0:
+    for i in 0 ..< delta:
+      var defaultVal: T
+      d.addLast(defaultVal)
+  else:
+    d.shrink(fromLast = delta)
+
+proc add*(pool: var AttestationPool,
+          attestation: Attestation,
+          beaconState: BeaconState) =
+  # The caller of this function is responsible for ensuring that
+  # the attestations will be given in a strictly slot increasing order:
+  doAssert attestation.data.slot.int >= pool.startingSlot
+
+  let slotIdxInPool = attestation.data.slot.int - pool.startingSlot
+  if slotIdxInPool >= pool.attestations.len:
+    pool.attestations.setLen(slotIdxInPool + 1)
+
+  pool.attestations[slotIdxInPool].add attestation
+
+iterator each*(pool: AttestationPool,
+               firstSlot, lastSlot: int): Attestation =
+  ## Both indices are treated inclusively
+  ## TODO: this should return a lent value
+  doAssert firstSlot <= lastSlot
+  for idx in countup(max(0, firstSlot - pool.startingSlot),
+                     min(pool.attestations.len - 1, lastSlot - pool.startingSlot)):
+    for attestation in pool.attestations[idx]:
+      yield attestation
+
+proc discardHistoryToSlot*(pool: var AttestationPool, slot: int) =
+  ## The index is treated inclusively
+  let slotIdx = slot - pool.startingSlot
+  if slotIdx < 0: return
+  pool.attestations.shrink(fromFirst = slotIdx + 1)
+
+proc getLatestAttestation*(pool: AttestationPool, validator: ValidatorRecord) =
+  discard
+
+proc getLatestAttestationTarget*() =
+  discard
+
+proc forkChoice*(pool: AttestationPool, oldHead, newBlock: BeaconBlock): bool =
+  # This will return true if the new block is accepted over the old head block
+  discard
+

beacon_chain/gossipsub_protocol.nim

@@ -0,0 +1,38 @@
+import
+  tables, sets,
+  asyncdispatch2, chronicles, rlp, eth_p2p, eth_p2p/rlpx
+
+type
+  TopicMsgHandler = proc(data: seq[byte]): Future[void]
+
+  GossibSubPeer = ref object
+    sentMessages: HashSet[string]
+
+  GossipSubNetwork = ref object
+    deliveredMessages: Table[Peer, HashSet[string]]
+    topicSubscribers: Table[string, seq[TopicMsgHandler]]
+
+p2pProtocol GossipSub(version = 1,
+                      shortName = "gss",
+                      peerState = GossibSubPeer,
+                      networkState = GossipSubNetwork):
+  # This is a very barebones emulation of the GossipSub protocol
+  # available in LibP2P:
+
+  proc interestedIn(peer: Peer, topic: string)
+  proc emit(peer: Peer, topic: string, msgId: string, data: openarray[byte])
+
+proc subscribeImpl(node: EthereumNode,
+                   topic: string,
+                   subscriber: TopicMsgHandler) =
+  discard
+
+proc broadcastImpl(node: EthereumNode, topic: string, data: seq[byte]) =
+  discard
+
+macro subscribe*(node: EthereumNode, topic: string, handler: untyped): untyped =
+  discard
+
+proc broadcast*(node: EthereumNode, topic: string, data: auto) {.async.} =
+  discard
+

beacon_chain/mainchain_monitor.nim

@@ -0,0 +1,29 @@
+import
+  asyncdispatch2, json_rpc/rpcclient,
+  spec/[datatypes, digest]
+
+type
+  MainchainMonitor* = object
+    gethAddress: string
+    gethPort: Port
+
+proc init*(T: type MainchainMonitor, gethAddress: string, gethPort: Port): T =
+  result.gethAddress = gethAddress
+  result.gethPort = gethPort
+
+proc start*(m: var MainchainMonitor) =
+  # TODO
+  # Start an async loop following the new blocks using the ETH1 JSON-RPC
+  # interface and keep an always-up-to-date receipt reference here
+  discard
+
+proc getBeaconBlockRef*(m: MainchainMonitor): Eth2Digest =
+  # This should be a simple accessor for the reference kept above
+  discard
+
+iterator getValidatorActions*(m: MainchainMonitor,
+                              fromBlock, toBlock: Eth2Digest): SpecialRecord =
+  # It's probably better if this doesn't return a SpecialRecord, but
+  # rather a more readable description of the change that can be packed
+  # in a SpecialRecord by the client of the API.
+  discard

beacon_chain/spec/beaconstate.nim

@@ -8,25 +8,22 @@
 import
   ./datatypes, ./digest, ./helpers, ./validator
 
-func get_shards_and_committees_for_slot*(state: BeaconState,
-                                         slot: uint64
-                                         ): seq[ShardAndCommittee] =
-  let earliest_slot_in_array = state.last_state_recalculation_slot - CYCLE_LENGTH
-  assert earliest_slot_in_array <= slot
-  assert slot < earliest_slot_in_array + CYCLE_LENGTH * 2
+func mod_get[T](arr: openarray[T], pos: Natural): T =
+  arr[pos mod arr.len]
 
-  return state.shard_and_committee_for_slots[int slot - earliest_slot_in_array]
-  # TODO, slot is a uint64; will be an issue on int32 arch.
-  #       Clarify with EF if light clients will need the beacon chain
+func get_shard_and_committees_idx*(state: BeaconState, slot: int): int =
+  # This replaces `get_shards_and_committees_for_slot` from the spec
+  # since in Nim, it's not currently efficient to create read-only
+  # accessors to expensive-to-copy members (such as sequences).
+  let earliest_slot_in_array = state.last_state_recalculation_slot.int - CYCLE_LENGTH
+  doAssert earliest_slot_in_array <= slot and
+           slot < earliest_slot_in_array + CYCLE_LENGTH * 2
+  return int(slot - earliest_slot_in_array)
 
-func get_block_hash*(state: BeaconState, current_block: BeaconBlock, slot: int): Eth2Digest =
-  let earliest_slot_in_array = current_block.slot.int - state.recent_block_hashes.len
-  assert earliest_slot_in_array <= slot
-  assert slot < current_block.slot.int
+proc get_shards_and_committees_for_slot*(state: BeaconState, slot: int): seq[ShardAndCommittee] =
+  return state.shard_and_committee_for_slots[state.get_shard_and_committees_idx(slot)]
 
-  return state.recent_block_hashes[slot - earliest_slot_in_array]
-
-func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord =
+func get_beacon_proposer_idx*(state: BeaconState, slot: int): int =
   ## From Casper RPJ mini-spec:
   ## When slot i begins, validator Vidx is expected
   ## to create ("propose") a block, which contains a pointer to some parent block
@@ -35,7 +32,20 @@ func get_beacon_proposer*(state: BeaconState, slot: uint64): ValidatorRecord =
   ## that have not yet been included into that chain.
   ##
   ## idx in Vidx == p(i mod N), pi being a random permutation of validators indices (i.e. a committee)
-  let
-    first_committee = get_shards_and_committees_for_slot(state, slot)[0].committee
-    index = first_committee[(slot mod len(first_committee).uint64).int]
-  state.validators[index]
+
+  # This replaces `get_beacon_proposer` from the spec since in Nim,
+  # it's not currently efficient to create read-only accessors to
+  # expensive-to-copy members (such as ValidatorRecord).
+
+  let idx = get_shard_and_committees_idx(state, slot)
+  return state.shard_and_committee_for_slots[idx][0].committee.mod_get(slot)
+
+func get_block_hash*(state: BeaconState,
+                     current_block: BeaconBlock,
+                     slot: int): Eth2Digest =
+  let earliest_slot_in_array = current_block.slot.int - state.recent_block_hashes.len
+  assert earliest_slot_in_array <= slot
+  assert slot < current_block.slot.int
+
+  return state.recent_block_hashes[slot - earliest_slot_in_array]
+

beacon_chain/spec/crypto.nim

@@ -10,8 +10,13 @@
 # hashed out. This layer helps isolate those chagnes.
 
 import
-  milagro_crypto
+  milagro_crypto, hashes
 
 type
-  Eth2PublicKey* = milagro_crypto.VerKey
-  Eth2Signature* = milagro_crypto.Signature
+  ValidatorPubKey* = milagro_crypto.VerKey
+  ValidatorPrivKey* = milagro_crypto.SigKey
+  ValidatorSig* = milagro_crypto.Signature
+
+template hash*(k: ValidatorPubKey|ValidatorPrivKey): Hash =
+  hash(k.getRaw)
+

beacon_chain/spec/datatypes.nim

@@ -69,13 +69,13 @@ type
     state_root*: Eth2Digest                        # State root
     attestations*: seq[AttestationRecord]          # Attestations
     specials*: seq[SpecialRecord]                  # Specials (e.g. logouts, penalties)
-    proposer_signature*: Eth2Signature             # Proposer signature
+    proposer_signature*: ValidatorSig              # Proposer signature
 
   AttestationRecord* = object
     data*: AttestationSignedData                   #
     attester_bitfield*: seq[byte]                  # Attester participation bitfield
     poc_bitfield*: seq[byte]                       # Proof of custody bitfield
-    aggregate_sig*: Eth2Signature                  # BLS aggregate signature
+    aggregate_sig*: ValidatorSig                   # BLS aggregate signature
 
   AttestationSignedData* = object
     slot*: uint64                                 # Slot number
@@ -93,7 +93,7 @@ type
     block_hash*: Eth2Digest                       # Block hash
 
   SpecialRecord* = object
-    kind*: SpecialRecordTypes                     # Kind
+    kind*: SpecialRecordType                      # Kind
     data*: seq[byte]                              # Data
 
   BeaconState* = object
@@ -124,7 +124,7 @@ type
     randao_mix*: Eth2Digest                                # RANDAO state
 
   ValidatorRecord* = object
-    pubkey*: Eth2PublicKey                        # Public key
+    pubkey*: ValidatorPubKey                      # Public key
     withdrawal_credentials*: Eth2Digest           # Withdrawal credentials
     randao_commitment*: Eth2Digest                # RANDAO commitment
     randao_skips*: uint64                         # Slot the proposer has skipped (ie. layers of RANDAO expected)
@@ -169,7 +169,7 @@ type
     Withdrawn = 4
     Penalized = 127
 
-  SpecialRecordTypes* {.pure.} = enum
+  SpecialRecordType* {.pure.} = enum
     Logout = 0
     CasperSlashing = 1
     RandaoChange = 2
@@ -191,3 +191,27 @@ type
     #   with room to spare.
     #
     #   Also, IntSets uses machine int size while we require int64 even on 32-bit platform.
+
+when true:
+  # TODO: Remove these once RLP serialization is no longer used
+  import nimcrypto, rlp
+  export append, read
+
+  proc append*(rlpWriter: var RlpWriter, value: ValidatorPubKey) =
+    discard
+
+  proc read*(rlp: var Rlp, T: type ValidatorPubKey): T {.inline.} =
+    discard
+
+  proc append*(rlpWriter: var RlpWriter, value: Uint24) =
+    discard
+
+  proc read*(rlp: var Rlp, T: type Uint24): T {.inline.} =
+    discard
+
+  proc append*(rlpWriter: var RlpWriter, value: ValidatorSig) =
+    discard
+
+  proc read*(rlp: var Rlp, T: type ValidatorSig): T {.inline.} =
+    discard
+

beacon_chain/spec/validator.nim

@@ -17,7 +17,7 @@ func min_empty_validator(validators: seq[ValidatorRecord], current_slot: uint64)
           return some(i)
 
 func add_validator*(validators: var seq[ValidatorRecord],
-                    pubkey: Eth2PublicKey,
+                    pubkey: ValidatorPubKey,
                     proof_of_possession: seq[byte],
                     withdrawal_credentials: Eth2Digest,
                     randao_commitment: Eth2Digest,

beacon_chain/ssz.nim

@@ -226,11 +226,11 @@ func hashSSZ*(x: enum): array[32, byte] =
   withHash:
     h.update [uint8 x]
 
-func hashSSZ*(x: Eth2Signature): array[32, byte] =
+func hashSSZ*(x: ValidatorSig): array[32, byte] =
   ## TODO - Warning ⚠️: not part of the spec
   ## as of https://github.com/ethereum/beacon_chain/pull/133/files
   ## This is a "stub" needed for BeaconBlock hashing
-  x.getraw().hash()
+  x.getRaw().hash()
 
 func hashSSZ*(x: AttestationRecord): array[32, byte] =
   ## TODO - Warning ⚠️: not part of the spec

beacon_chain/state_transition.nim

@@ -25,8 +25,6 @@ import
   intsets, endians, nimcrypto,
   milagro_crypto # nimble install https://github.com/status-im/nim-milagro-crypto@#master
 
-
-
 func process_block*(active_state: BeaconState, crystallized_state: BeaconState, blck: BeaconBlock, slot: uint64) =
   # TODO: non-attestation verification parts of per-block processing
 
@@ -44,7 +42,7 @@ func process_block*(active_state: BeaconState, crystallized_state: BeaconState,
 
     # Let attestation_indices be get_shards_and_committees_for_slot(crystallized_state, slot)[x], choosing x so that attestation_indices.shard_id equals the shard_id value provided to find the set of validators that is creating this attestation record.
     let attestation_indices = block:
-      let shard_and_committees = get_shards_and_committees_for_slot(crystallized_state, slot)
+      let shard_and_committees = get_shards_and_committees_for_slot(crystallized_state, slot.int)
       var
         x = 1
         record_creator = shard_and_committees[0]
@@ -53,11 +51,10 @@ func process_block*(active_state: BeaconState, crystallized_state: BeaconState,
         inc x
       record_creator
 
-    # Verify that len(attester_bitfield) == ceil_div8(len(attestation_indices)), where ceil_div8 = (x + 7) // 8. Verify that bits len(attestation_indices).... and higher, if present (i.e. len(attestation_indices) is not a multiple of 8), are all zero
-    doAssert attestation.attester_bitfield.len == attestation_indices.committee.len
+    # TODO: Verify that len(attester_bitfield) == ceil_div8(len(attestation_indices)), where ceil_div8 = (x + 7) // 8. Verify that bits len(attestation_indices).... and higher, if present (i.e. len(attestation_indices) is not a multiple of 8), are all zero
 
     # Derive a group public key by adding the public keys of all of the attesters in attestation_indices for whom the corresponding bit in attester_bitfield (the ith bit is (attester_bitfield[i // 8] >> (7 - (i %8))) % 2) equals 1
-    var agg_pubkey: Eth2PublicKey
+    var agg_pubkey: ValidatorPubKey
     var empty = true
     for attester_idx in attestation_indices.committee:
       # TODO re-enable, but currently this whole function's a nonfunctional stub

beacon_chain/sync_protocol.nim

@@ -0,0 +1,93 @@
+import
+  options,
+  chronicles, rlp, asyncdispatch2, ranges/bitranges, eth_p2p, eth_p2p/rlpx,
+  spec/[datatypes, crypto, digest]
+
+type
+  ValidatorChangeLogEntry* = object
+    case kind*: ValidatorSetDeltaFlags
+    of Entry:
+      pubkey: ValidatorPubKey
+    else:
+      index: uint32
+
+  ValidatorSet = seq[ValidatorRecord]
+
+p2pProtocol BeaconSync(version = 1,
+                       shortName = "bcs"):
+  requestResponse:
+    proc getValidatorChangeLog(peer: Peer, changeLogHead: Eth2Digest)
+
+    proc validatorChangeLog(peer: Peer,
+                            signedBlock: BeaconBlock,
+                            beaconState: BeaconState,
+                            added: openarray[ValidatorPubKey],
+                            removed: openarray[uint32],
+                            order: seq[byte])
+
+template `++`(x: var int): int =
+  let y = x
+  inc x
+  y
+
+type
+  # A bit shorter names for convenience
+  ChangeLog = BeaconSync.validatorChangeLog
+  ChangeLogEntry = ValidatorChangeLogEntry
+
+func validate*(log: ChangeLog): bool =
+  # TODO:
+  # Assert that the number of raised bits in log.order (a.k.a population count)
+  # matches the number of elements in log.added
+  # https://en.wikichip.org/wiki/population_count
+  return true
+
+iterator changes*(log: ChangeLog): ChangeLogEntry =
+  var
+    bits = log.added.len + log.removed.len
+    addedIdx = 0
+    removedIdx = 0
+
+  template nextItem(collection): auto =
+    let idx = `collection Idx`
+    inc `collection Idx`
+    log.collection[idx]
+
+  for i in 0 ..< bits:
+    yield if log.order.getBit(i):
+      ChangeLogEntry(kind: Entry, pubkey: nextItem(added))
+    else:
+      ChangeLogEntry(kind: Exit, index: nextItem(removed))
+
+proc getValidatorChangeLog*(node: EthereumNode, changeLogHead: Eth2Digest):
+                            Future[(Peer, ChangeLog)] {.async.} =
+  while true:
+    let peer = node.randomPeerWith(BeaconSync)
+    if peer == nil: return
+
+    let res = await peer.getValidatorChangeLog(changeLogHead, timeout = 1)
+    if res.isSome:
+      return (peer, res.get)
+
+proc applyValidatorChangeLog*(log: ChangeLog,
+                              outBeaconState: var BeaconState): bool =
+  # TODO:
+  #
+  # 1. Validate that the signedBlock state root hash matches the
+  #    provided beaconState
+  #
+  # 2. Validate that the applied changelog produces the correct
+  #    new change log head
+  #
+  # 3. Check that enough signatures from the known validator set
+  #    are present
+  #
+  # 4. Apply all changes to the validator set
+  #
+
+  outBeaconState.last_finalized_slot =
+    log.signedBlock.slot div CYCLE_LENGTH
+
+  outBeaconState.validator_set_delta_hash_chain =
+    log.beaconState.validator_set_delta_hash_chain
+

beacon_chain/time.nim

@@ -0,0 +1,46 @@
+import
+  random,
+  asyncdispatch2,
+  spec/datatypes
+
+type
+  Timestamp = uint64 # Unix epoch timestamp in millisecond resolution
+
+var
+  detectedClockDrift: int64
+
+proc timeSinceGenesis*(s: BeaconState): Timestamp =
+  Timestamp(int64(fastEpochTime() - s.genesis_time * 1000) -
+            detectedClockDrift)
+
+template toSlot*(t: Timestamp): int =
+  int(t div uint64(SLOT_DURATION * 1000))
+
+template slotStart*(s: BeaconState, slot: int): Timestamp =
+  (s.genesis_time + uint64(slot * SLOT_DURATION)) * 1000
+
+template slotMiddle*(s: BeaconState, slot: int): Timestamp =
+  s.slotStart(slot) + SLOT_DURATION * 500
+
+template slotEnd*(s: BeaconState, slot: int): Timestamp =
+  s.slotStart(slot + 1)
+
+proc randomTimeInSlot*(s: BeaconState,
+                       slot: Natural,
+                       interval: HSlice[float, float]): Timestamp =
+  ## Returns a random moment within the slot.
+  ## The interval must be a sub-interval of [0..1].
+  ## Zero marks the begginning of the slot and One marks the end.
+  s.slotStart(slot) + Timestamp(rand(interval) * float(SLOT_DURATION * 1000))
+
+proc slotDistanceFromNow*(s: BeaconState): int64 =
+  ## Returns how many slots have passed since a particular BeaconState was finalized
+  int64(s.timeSinceGenesis() div (SLOT_DURATION * 1000)) - int64(s.last_finalized_slot)
+
+proc syncrhronizeClock*() {.async.} =
+  ## This should determine the offset of the local clock against a global
+  ## trusted time (e.g. it can be obtained from multiple time servers).
+
+  # TODO: implement this properly
+  detectedClockDrift = 0
+

beacon_chain/trusted_state_snapshots.nim

@@ -0,0 +1,31 @@
+import
+  asyncdispatch2,
+  spec/datatypes, beacon_chain_db
+
+const
+  WEAK_SUBJECTVITY_PERIOD* = 4 * 30 * 24 * 60 * 60 div SLOT_DURATION
+    # TODO: This needs revisiting.
+    # Why was the validator WITHDRAWAL_PERIOD altered in the spec?
+
+proc obtainTrustedStateSnapshot*(db: BeaconChainDB): Future[BeaconState] {.async.} =
+  # In case our latest state is too old, we must obtain a recent snapshot
+  # of the state from a trusted location. This is explained in detail here:
+  # https://notes.ethereum.org/oaQV3IF5R2qlJuW-V1r1ew#Beacon-chain-sync
+
+  # TODO: implement this:
+  #
+  # 1. Specify a large set of trusted state signees
+  # (perhaps stored in a config file)
+  #
+  # 2. Download a signed state hash from a known location
+  # (The known location can be either a HTTPS host or a DHT record)
+  #
+  # 3. Check that enough of the specified required signatures are present
+  #
+  # 4. Download a snapshot file from a known location
+  # (or just obtain it from the network using the ETH protocols)
+  #
+  # 5. Check that the state snapshot hash is correct and save it in the DB.
+
+  discard
+

beacon_chain/validator_pool.nim

@@ -0,0 +1,66 @@
+import
+  tables, random,
+  asyncdispatch2,
+  spec/[datatypes, crypto]
+
+type
+  ValidatorKind = enum
+    inProcess
+    remote
+
+  ValidatorConnection = object
+
+  RandaoSecret = seq[byte]
+
+  AttachedValidator* = ref object
+    idx*: int
+    case kind: ValidatorKind
+    of inProcess:
+      privKey: ValidatorPrivKey
+      randaoSecret: RandaoSecret
+    else:
+      connection: ValidatorConnection
+
+  ValidatorPool* = object
+    validators: Table[ValidatorPubKey, AttachedValidator]
+
+proc init*(T: type ValidatorPool): T =
+  result.validators = initTable[ValidatorPubKey, AttachedValidator]()
+
+proc addLocalValidator*(pool: var ValidatorPool,
+                        idx: int,
+                        pubKey: ValidatorPubKey,
+                        privKey: ValidatorPrivKey,
+                        randaoSecret: RandaoSecret) =
+  pool.validators[pubKey] = AttachedValidator(idx: idx,
+                                              kind: inProcess,
+                                              privKey: privKey,
+                                              randaoSecret: randaoSecret)
+
+proc getValidator*(pool: ValidatorPool,
+                   validatorKey: ValidatorPubKey): AttachedValidator =
+  pool.validators.getOrDefault(validatorKey)
+
+proc signBlockProposal*(v: AttachedValidator,
+                        proposal: ProposalSignedData): Future[ValidatorSig] {.async.} =
+  if v.kind == inProcess:
+    await sleepAsync(1)
+    # TODO:
+    # return sign(proposal, v.privKey)
+  else:
+    # TODO:
+    # send RPC
+    discard
+
+proc signAttestation*(v: AttachedValidator,
+                      attestation: AttestationSignedData): Future[ValidatorSig] {.async.} =
+  # TODO: implement this
+  if v.kind == inProcess:
+    await sleepAsync(1)
+    # TODO:
+    # return sign(proposal, v.privKey)
+  else:
+    # TODO:
+    # send RPC
+    discard
+