Add more flesh to the skeleton :)

beacon_chain/beacon_node.nim

@@ -1,9 +1,9 @@
 import
   os, net,
   asyncdispatch2, confutils, eth_p2p, eth_keys,
-  conf, datatypes, time, beacon_chain_db, validator_pool,
+  private/helpers, conf, datatypes, time, fork_choise,
-  sync_protocol, gossipsub_protocol, trusted_state_snapshots,
+  beacon_chain_db, validator_pool, mainchain_monitor,
-  private/helpers
+  sync_protocol, gossipsub_protocol, trusted_state_snapshots
 
 type
   BeaconNode* = ref object
@@ -12,12 +12,17 @@ type
     db*: BeaconChainDB
     config*: Configuration
     keys*: KeyPair
-    attachedValidators: Table[BLSPublicKey, AttachedValidator]
+    attachedValidators: ValidatorPool
+    attestations: AttestationPool
+    headBlock: BeaconBlock
+    mainchainMonitor: MainchainMonitor
 
 const
   version = "v0.1" # read this from the nimble file
   clientId = "nimbus beacon node " & version
-  topicBeaconBlocks = "ethereum/2.1/beacon_blocks"
+
+  topicBeaconBlocks = "ethereum/2.1/beacon_chain/blocks"
+  topicAttestations = "ethereum/2.1/beacon_chain/attestations"
 
 proc ensureNetworkKeys*(dataDir: string): KeyPair =
   # TODO:
@@ -73,32 +78,88 @@ proc addLocalValidators*(node: BeaconNode) =
     # 3. Add the validators to node.attachedValidators
     discard
 
+proc getAttachedValidator(node: BeaconNode, idx: int): AttachedValidator =
+  let validatorKey = node.beaconState.validators[idx].pubkey
+  return node.attachedValidators.getValidator(validatorKey)
+
+proc makeAttestationCallback(node: BeaconNode,
+                             validator: AttachedValidator): auto =
+  proc makeAttestation {.async.} =
+    var attestation: Attestation
+    attestation.validator = validator.idx
+
+    # TODO: Populate attestation.data
+
+    attestation.signature = await validator.signAttestation(attestation.data)
+    await node.network.broadcast(topicAttestations, attestation)
+
+  return proc =
+    asyncCheck makeAttestation
+
+proc proposeBlockCallback(node: BeaconNode,
+                          validator: AttachedValidator): auto =
+  proc proposeBlock {.async.} =
+    var proposal: BeaconBlock
+
+    # TODO:
+    # 1. Produce a RANDAO reveal from attachedVadalidator.randaoSecret
+    # and its matching ValidatorRecord.
+
+    # 2. Get ancestors from the beacon_db
+
+    # 3. Calculate the correct state hash
+
+    proposal.candidate_pow_receipt_root =
+      node.mainchainMonitor.getBeaconBlockRef()
+
+    for a in node.attestations.each(firstSlot = node.headBlock.slot + 1,
+                                    lastSlot = slot - MIN_ATTESTATION_INCLUSION_DELAY):
+      proposal.attestations.add a
+      # TODO: this is not quite right,
+      # the attestations from individual validators have to be merged.
+
+    for r in node.mainchainMonitor.getValidatorActions():
+      proposal.specials.add r
+
+    var signedData: ProposalSignedData
+    # TODO: populate the signed data
+
+    proposal.proposer_signature = await validator.signBlockProposal(signedData)
+    await node.network.broadcast(topicProposals, proposal)
+
+  return proc =
+    asyncCheck proposeBlock
+
 proc scheduleCycleActions(node: BeaconNode)
   ## This schedules the required block proposals and
   ## attestations from our attached validators.
-  let cycle_start = node.last_state_recalculation_slot
+  let cycleStart = node.last_state_recalculation_slot
 
-  # Schedule block proposals
   for i in 0 ..< CYCLE_LENGTH:
+    # Schedule block proposals
     let
-      proposer_idx = get_beacon_proposer_idx(node.beaconState, cycle_start + i)
+      slot = cycleStart + i
-      proposer_key = node.beaconState.validators[proposer_idx].pubkey
+      proposerIdx = get_beacon_proposer_idx(node.beaconState, slot)
-      attached_validator = node.attachedValidators.getAttachedValidator(proposer_key)
+      attachedValidator = node.getAttachedValidator(proposerIdx)
-    
-    if attached_validator != nil:
-      proc proposeBlock =
-        # TODO
-        discard
 
+    if attachedValidator != nil:
       # TODO:
       # Warm-up the proposer earlier to try to obtain previous
       # missing blocks if necessary
 
-      addTimer slotMiddle(cycle_start + i), proposeBlock
+      addTimer slotStart(slot),
+               proposeBlockCallback(node, attachedValidator)
 
-  # Schedule attestations
+    # Schedule attestations
-  # TODO:
+    let
-  # Similar to the above, but using `get_shard_and_committees_idx`
+      committeesIdx = get_shard_and_committees_idx(node.beaconState, slot)
+
+      for shard in node.beaconState.shard_and_committee_for_slots[committees_idx]:
+        for validatorIdx in shard.committee:
+          let attachedValidator = node.getAttachedValidator(validatorIdx)
+          if attachedValidator != nil:
+            addTimer slotMiddle(slot),
+                     makeAttestationCallback(node, attachedValidator)
 
 proc processBlocks*(node: BeaconNode) {.async.} =
   node.scheduleCycleActions()
@@ -107,9 +168,22 @@ proc processBlocks*(node: BeaconNode) {.async.} =
     # TODO:
     #
     # 1. Check for missing blocks and obtain them
+    #
+    # 2. Apply fork-choice rule (update node.headBlock)
+    #
+    # 3. Peform block processing / state recalculation / etc
+    #
 
     if b.slot mod CYCLE_LENGTH == 0:
       node.scheduleCycleActions()
+      node.attestations.discardHistoryToSlot(b.slot)
+
+  node.network.subscribe(topicAttestations) do (a: Attestation):
+    # TODO
+    #
+    # 1. Validate the attestation
+
+    node.attestations.add(a, node.beaconState)
 
 when isMainModule:
   let conf = Configuration.load()

beacon_chain/fork_choice.nim

@@ -0,0 +1,56 @@
+import
+  deque,
+  datatypes
+
+type
+  Attestation* = object
+    validator*: int
+    data*: AttestationSignedData
+    signature*: BLSsig
+
+  AttestationPool* = object
+    attestations: deque[seq[Attestation]]
+    startingSlot: int
+
+proc init*(T: type AttestationPool, startingSlot: int): T =
+  result.attestationsPerSlot = initDeque[seq[Attestation]]()
+  result.startingSlot = startingSlot
+
+proc add*(pool: var AttestationPool,
+          attestation: Attestation,
+          beaconState: BeaconState) =
+  let slotIdxInPool = attestation.slot - pool.startingSlot
+  # The caller of this function is responsible for ensuring that
+  # the attestations will be given in a strictly slot increasing order:
+  doAssert slotIdxInPool < 0
+
+  if slotIdxInPool >= pool.attestations.len:
+    pool.attestations.setLen(slotIdxInPool + 1)
+  pool.attestations[slotIdxInPool].add attestation
+
+iterator each*(pool: AttestationPool,
+               firstSlot, lastSlot: int): Attestation =
+  ## Both indices are treated inclusively
+  ## TODO: this should return a lent value
+  doAssert firstSlot <= lastSlot
+  for idx in countup(max(0, firstSlot - pool.startingSlot),
+                     min(pool.attestation.len - 1, lastSlot - pool.startingSlot)):
+    for attestation in pool.attestations[idx]:
+      yield attestation
+
+proc discardHistoryToSlot*(pool: var AttestationPool, slot: int) =
+  ## The index is treated inclusively
+  let slotIdx = slot - pool.startingSlot
+  if slotIdx < 0: return
+  pool.attestation.shrink(fromFirst = slotIdx + 1)
+
+proc getLatestAttestation*(pool: AttestationPool, validator: ValidatorRecord) =
+  discard
+
+proc getLatestAttestationTarget*() =
+  discard
+
+proc forkChoice*(pool: AttestationPool, oldHead, newBlock: BeaconBlock): bool =
+  # This will return true if the new block is accepted over the old head block
+  discard
+

beacon_chain/gossipsub_protocol.nim

@@ -1,7 +1,32 @@
 import
+  tables, sets,
   eth_p2p, eth_p2p/rlpx
 
 type
+  TopicMsgHandler = proc(data: seq[byte]): Future[void]
+
+  GossipSubNetwork = type
+    deliveredMessages: Table[Peer, HashSet[string]]
+    topicSubscribers: Table[string, seq[TopicMsgHandler]]
 
 protocol GossipSub(version = 1):
+  # This is a very barebones emulation of the GossipSub protocol
+  # available in LibP2P:
+
+  proc interestedIn(topic: string)
+  proc emit(topic: string, msgId: string, data: openarray[byte])
+
+proc subscribeImpl(node: EthereumNode,
+                   topic: string,
+                   subscriber: TopicMsgHandler) =
+  discard
+
+proc broadcastImpl(node: EthereumNode, topic: string, data: seq[byte]) =
+  discard
+
+macro subscribe*(node: EthereumNode, topic: string, handler: body): untyped =
+  discard
+
+proc broadcast*(node: EthereumNode, topic: string, data: auto) =
+  discard
 

beacon_chain/mainchain_monitor.nim

@@ -0,0 +1,31 @@
+import
+  asyncdispatch2, json_rpc,
+  datatypes
+
+type
+  MainchainMonitor* = object
+    gethAddress: string
+    gethPort: Port
+
+
+
+proc init*(T: type MainchainMonitor, gethAddress: string, gethPort: Port): T =
+  result.gethAddress = gethAddress
+  result.gethPort = gethPort
+
+proc start*(m: var MainchainMonitor) =
+  # TODO
+  # Start an async loop following the new blocks using the ETH1 JSON-RPC
+  # interface and keep an always-up-to-date receipt reference here
+  discard
+
+proc getBeaconBlockRef*(m: MainchainMonitor): Blake2_256_Digest =
+  # This should be a simple accessor for the reference kept above
+  discard
+
+iterator getValidatorActions*(fromBlock,
+                              toBlock: Blake2_256_Digest): SpecialRecord =
+  # It's probably better if this doesn't return a SpecialRecord, but
+  # rather a more readable description of the change that can be packed
+  # in a SpecialRecord by the client of the API.
+  discard

beacon_chain/private/helpers.nim

@@ -98,7 +98,7 @@ func mod_get[T](arr: openarray[T], pos: Natural): T =
   arr[pos mod arr.len]
 
 func get_shard_and_committees_idx*(state: BeaconState, slot: uint64): int =
-  # This replaces `get_shards_and_committees_for_slot` from the spec,
+  # This replaces `get_shards_and_committees_for_slot` from the spec
   # since in Nim, it's not currently efficient to create read-only
   # accessors to expensive-to-copy members (such as sequences).
   let earliest_slot_in_array = state.last_state_recalculation_slot - CYCLE_LENGTH
@@ -107,9 +107,9 @@ func get_shard_and_committees_idx*(state: BeaconState, slot: uint64): int =
   return int(slot - earliest_slot_in_array)
 
 func get_beacon_proposer_idx*(state: BeaconState, slot: int): int =
-  # This replaces `get_beacon_proposer` from the spec,
+  # This replaces `get_beacon_proposer` from the spec since in Nim,
-  # since in Nim, it's not currently efficient to create read-only
+  # it's not currently efficient to create read-only accessors to
-  # accessors to expensive-to-copy members (such as ValidatorRecord).
+  # expensive-to-copy members (such as ValidatorRecord).
   let idx = get_shard_and_committees_idx(state, slot)
   return state.shard_and_committee_for_slots[idx][0].committee.mod_get(slot)
 

beacon_chain/validator_pool.nim

@@ -13,11 +13,12 @@ type
   RandaoValue = seq[bytes]
 
   AttachedValidator* = ref object
-    validatorSlot: int
+    idx*: int
     case kind: ValidatorKind
     of inProcess:
       randaoValue: RandaoValue
       privKey: BLSPrivateKey
+      randaoSecret: seq[bytes]
     else:
       connection: ValidatorConnection
 
@@ -32,8 +33,8 @@ proc addLocalValidator*(pool: var ValidatorPool,
                         privKey: BLSPrivateKey) =
   discard
 
-proc getAttachedValidator*(pool: ValidatorPool,
+proc getValidator*(pool: ValidatorPool,
-                           validatorKey: BLSPublicKey): AttachedValidator =
+                   validatorKey: BLSPublicKey): AttachedValidator =
   pool.validatators.getOrDefault(validatorKey)
 
 proc signBlockProposal*(v: AttachedValidator,