spec updates

* first attestation created!
* add hash_tree_root_final that returns an Eth2Digest
* hits the first real blocking spec bug :(

beacon_chain/beacon_node.nim

@@ -162,7 +162,7 @@ proc proposeBlock(node: BeaconNode,
   var signedData: ProposalSignedData
   signedData.slot = node.beaconState.slot
   signedData.shard = BEACON_CHAIN_SHARD_NUMBER
-  signedData.blockRoot.data = hash_tree_root(proposal)
+  signedData.blockRoot = hash_tree_root_final(proposal)
 
   proposal.signature = await validator.signBlockProposal(signedData)
   await node.network.broadcast(topicBeaconBlocks, proposal)

beacon_chain/spec/beaconstate.nim

@@ -17,9 +17,10 @@ func process_deposit(state: var BeaconState,
                      withdrawal_credentials: Eth2Digest,
                      randao_commitment: Eth2Digest): Uint24 =
   ## Process a deposit from Ethereum 1.0.
-  let msg = hash_tree_root((pubkey, withdrawal_credentials, randao_commitment))
+  let msg = hash_tree_root_final(
+    (pubkey, withdrawal_credentials, randao_commitment))
   assert bls_verify(
-    pubkey, msg, proof_of_possession,
+    pubkey, msg.data, proof_of_possession,
     get_domain(state.fork_data, state.slot, DOMAIN_DEPOSIT))
 
   let validator_pubkeys = mapIt(state.validator_registry, it.pubkey)
@@ -249,9 +250,7 @@ func get_attestation_participants*(state: BeaconState,
     # TODO investigate functional library / approach to help avoid loop bugs
     assert len(participation_bitfield) == ceil_div8(len(snc.committee))
     for i, vindex in snc.committee:
-      let
+      if bitIsSet(participation_bitfield, i):
-        bit = (participation_bitfield[i div 8] shr (7 - (i mod 8))) mod 2
-      if bit == 1:
         result.add(vindex)
     return # found the shard, we're done
 
@@ -326,10 +325,10 @@ proc checkAttestation*(state: BeaconState, attestation: Attestation): bool =
       participants, state.validator_registry[it].pubkey))
 
   # Verify that aggregate_signature verifies using the group pubkey.
-  let msg = hash_tree_root(attestation.data)
+  let msg = hash_tree_root_final(attestation.data)
 
   if not bls_verify(
-        group_public_key, @msg & @[0'u8], attestation.aggregate_signature,
+        group_public_key, @(msg.data) & @[0'u8], attestation.aggregate_signature,
         get_domain(state.fork_data, attestation.data.slot, DOMAIN_ATTESTATION)
       ):
     warn("Invalid attestation group signature")

beacon_chain/spec/crypto.nim

@@ -27,7 +27,7 @@ template hash*(k: ValidatorPubKey|ValidatorPrivKey): Hash =
 func pubKey*(pk: ValidatorPrivKey): ValidatorPubKey = fromSigKey(pk)
 
 func bls_aggregate_pubkeys*(keys: openArray[ValidatorPubKey]): ValidatorPubKey =
-  var empty = false
+  var empty = true
   for key in keys:
     if empty:
       result = key

beacon_chain/spec/datatypes.nim

@@ -43,6 +43,8 @@ import
 # Eventually, we could also differentiate between user/tainted data and
 # internal state that's gone through sanity checks already.
 
+# TODO Many of these constants should go into a config object that can be used
+#      to run.. well.. a chain with different constants!
 const
   SHARD_COUNT* = 1024 ##\
   ## Number of shards supported by the network - validators will jump around

beacon_chain/spec/helpers.nim

@@ -9,6 +9,14 @@
 
 import ./datatypes, ./digest, sequtils, math
 
+# TODO spec candidate? there's bits in nim-ranges but that one has some API
+#      issues regarding bit endianess that need resolving..
+func bitIsSet*(bitfield: openArray[byte], index: int): bool =
+  (bitfield[index div 8] shr byte(7 - (index mod 8))) mod 2 > 0'u8
+
+func bitSet*(bitfield: var openArray[byte], index: int) =
+  bitfield[index div 8] = bitfield[index div 8] or 1'u8 shl (7 - (index mod 8))
+
 func mod_get[T](arr: openarray[T], pos: Natural): T =
   arr[pos mod arr.len]
 

beacon_chain/spec/validator.nim

@@ -71,12 +71,12 @@ func get_new_validator_registry_delta_chain_tip*(
     flag: ValidatorSetDeltaFlags): Eth2Digest =
   ## Compute the next hash in the validator registry delta hash chain.
 
-  Eth2Digest(data: hash_tree_root(ValidatorRegistryDeltaBlock(
+  hash_tree_root_final(ValidatorRegistryDeltaBlock(
     latest_registry_delta_root: current_validator_registry_delta_chain_tip,
     validator_index: index,
     pubkey: pubkey,
     flag: flag
-  )))
+  ))
 
 func get_effective_balance*(validator: ValidatorRecord): uint64 =
     min(validator.balance, MAX_DEPOSIT * GWEI_PER_ETH)

beacon_chain/ssz.nim

@@ -306,6 +306,13 @@ func hash_tree_root*(x: ValidatorSig): array[32, byte] =
   ## This is a "stub" needed for BeaconBlock hashing
   x.getRaw().hash()
 
+func hash_tree_root_final*(x: object|tuple): Eth2Digest =
+  # TODO suggested for spec:
+  # https://github.com/ethereum/eth2.0-specs/issues/276
+  # only for objects now, else the padding would have to be implemented - not
+  # needed yet..
+  Eth2Digest(data: hash_tree_root(x))
+
 # ################### Tree hash ###################################
 
 func merkleHash[T](lst: openArray[T]): array[32, byte] =

beacon_chain/state_transition.nim

@@ -62,14 +62,14 @@ func verifyProposerSignature(state: BeaconState, blck: BeaconBlock): bool =
     signed_data = ProposalSignedData(
       slot: state.slot,
       shard: BEACON_CHAIN_SHARD_NUMBER,
-      block_root: Eth2Digest(data: hash_tree_root(blck_without_sig))
+      block_root: hash_tree_root_final(blck_without_sig)
     )
-    proposal_hash = hash_tree_root(signed_data)
+    proposal_hash = hash_tree_root_final(signed_data)
     proposer_index = get_beacon_proposer_index(state, state.slot)
 
   bls_verify(
     state.validator_registry[proposer_index].pubkey,
-    proposal_hash, blck.signature,
+    proposal_hash.data, blck.signature,
     get_domain(state.fork_data, state.slot, DOMAIN_PROPOSAL))
 
 func processRandao(
@@ -132,7 +132,7 @@ proc processProposerSlashings(state: var BeaconState, blck: BeaconBlock): bool =
     let proposer = addr state.validator_registry[proposer_slashing.proposer_index]
     if not bls_verify(
         proposer.pubkey,
-        hash_tree_root(proposer_slashing.proposal_data_1),
+        hash_tree_root_final(proposer_slashing.proposal_data_1).data,
         proposer_slashing.proposal_signature_1,
         get_domain(
           state.fork_data, proposer_slashing.proposal_data_1.slot,
@@ -141,7 +141,7 @@ proc processProposerSlashings(state: var BeaconState, blck: BeaconBlock): bool =
       return false
     if not bls_verify(
         proposer.pubkey,
-        hash_tree_root(proposer_slashing.proposal_data_2),
+        hash_tree_root_final(proposer_slashing.proposal_data_2).data,
         proposer_slashing.proposal_signature_2,
         get_domain(
           state.fork_data, proposer_slashing.proposal_data_2.slot,
@@ -469,6 +469,7 @@ func processEpoch(state: var BeaconState) =
       state.slot <= it.data.slot + 2 * EPOCH_LENGTH and
       it.data.slot + EPOCH_LENGTH < state.slot)
 
+  let
     previous_epoch_attesters =
       get_attesters(state, previous_epoch_attestations)
 
@@ -755,7 +756,7 @@ func processEpoch(state: var BeaconState) =
     )
 
 proc verifyStateRoot(state: BeaconState, blck: BeaconBlock): bool =
-  let state_root = Eth2Digest(data: hash_tree_root(state))
+  let state_root = hash_tree_root_final(state)
   if state_root != blck.state_root:
     warn("Block: root verification failed",
       block_state_root = blck.state_root, state_root)

beacon_chain/validator_keygen.nim

@@ -52,8 +52,8 @@ proc main() =
       genSingleValidator(outPath / &"validator-{i:02}.json")
 
     let withdrawalCredentials = makeFakeHash(i)
-    let proofOfPossession = signMessage(privkey, hash_tree_root(
+    let proofOfPossession = signMessage(privkey, hash_tree_root_final(
-      (pubKey, withdrawalCredentials, randaoCommitment)))
+      (pubKey, withdrawalCredentials, randaoCommitment)).data)
 
     startupData.validatorDeposits.add Deposit(
       deposit_data: DepositData(

beacon_chain/validator_pool.nim

@@ -47,10 +47,10 @@ proc signBlockProposal*(v: AttachedValidator,
                         proposal: ProposalSignedData): Future[ValidatorSig] {.async.} =
   if v.kind == inProcess:
     await sleepAsync(1)
-    let proposalRoot = hash_tree_root(proposal)
+    let proposalRoot = hash_tree_root_final(proposal)
 
     # TODO: Should we use proposalRoot as data, or digest in regards to signature?
-    return signMessage(v.privKey, proposalRoot)
+    return signMessage(v.privKey, proposalRoot.data)
   else:
     # TODO:
     # send RPC
@@ -62,9 +62,9 @@ proc signAttestation*(v: AttachedValidator,
   if v.kind == inProcess:
     await sleepAsync(1)
 
-    let attestationRoot = hash_tree_root(attestation)
+    let attestationRoot = hash_tree_root_final(attestation)
     # TODO: Should we use attestationRoot as data, or digest in regards to signature?
-    return signMessage(v.privKey, attestationRoot)
+    return signMessage(v.privKey, attestationRoot.data)
   else:
     # TODO:
     # send RPC

research/state_sim.nim

@@ -27,7 +27,7 @@ proc transition(
 
   var
     state = genesisState
-    latest_block_root = Eth2Digest(data: hash_tree_root(genesisBlock))
+    latest_block_root = hash_tree_root_final(genesisBlock)
 
   for i in 0..<slots:
     if state.slot mod json_interval.uint64 == 0:
@@ -36,8 +36,8 @@ proc transition(
     else:
       write(stdout, ".")
 
-    latest_block_root = Eth2Digest(data: hash_tree_root(
+    latest_block_root = hash_tree_root_final(
-      addBlock(state, latest_block_root, BeaconBlockBody())))
+      addBlock(state, latest_block_root, BeaconBlockBody()))
 
     flushFile(stdout)
 

tests/test_state_transition.nim

@@ -26,7 +26,7 @@ suite "Block processing":
     let
       state = genesisState
       proposer_index = getNextBeaconProposerIndex(state)
-      previous_block_root = Eth2Digest(data: hash_tree_root(genesisBlock))
+      previous_block_root = hash_tree_root_final(genesisBlock)
       new_state = updateState(
         state, previous_block_root, none(BeaconBlock), {})
     check:
@@ -42,7 +42,7 @@ suite "Block processing":
     let
       state = genesisState
       proposer_index = getNextBeaconProposerIndex(state)
-      previous_block_root = Eth2Digest(data: hash_tree_root(genesisBlock))
+      previous_block_root = hash_tree_root_final(genesisBlock)
       new_block = makeBlock(state, previous_block_root, BeaconBlockBody())
       new_state = updateState(
         state, previous_block_root, some(new_block), {})
@@ -59,7 +59,7 @@ suite "Block processing":
   test "Passes through epoch update, no block":
     var
       state = genesisState
-      previous_block_root = Eth2Digest(data: hash_tree_root(genesisBlock))
+      previous_block_root = hash_tree_root_final(genesisBlock)
 
     for i in 1..EPOCH_LENGTH.int:
       let new_state = updateState(
@@ -74,7 +74,7 @@ suite "Block processing":
   test "Passes through epoch update, empty block":
     var
       state = genesisState
-      previous_block_root = Eth2Digest(data: hash_tree_root(genesisBlock))
+      previous_block_root = hash_tree_root_final(genesisBlock)
 
     for i in 1..EPOCH_LENGTH.int:
       var new_block = makeBlock(state, previous_block_root, BeaconBlockBody())
@@ -86,7 +86,49 @@ suite "Block processing":
         new_state.block_ok
       state = new_state.state
       if new_state.block_ok:
-        previous_block_root = Eth2Digest(data: hash_tree_root(new_block))
+        previous_block_root = hash_tree_root_final(new_block)
 
     check:
       state.slot == genesisState.slot + EPOCH_LENGTH
+
+  test "Attestation gets processed at epoch":
+    var
+      state = genesisState
+      previous_block_root = hash_tree_root_final(genesisBlock)
+
+    # Slot 0 is a finalized slot - won't be making attestations for it..
+    state = updateState(
+        state, previous_block_root, none(BeaconBlock), {}).state
+
+    let
+      # Create an attestation for slot 1 signed by the only attester we have!
+      attestation = makeAttestation(
+        state, previous_block_root,
+        state.shard_committees_at_slots[state.slot][0].committee[0])
+
+    # Some time needs to pass before attestations are included - this is
+    # to let the attestation propagate properly to interested participants
+    while state.slot < MIN_ATTESTATION_INCLUSION_DELAY + 1:
+      state = updateState(
+        state, previous_block_root, none(BeaconBlock), {}).state
+
+    let
+      new_block = makeBlock(state, previous_block_root, BeaconBlockBody(
+        attestations: @[attestation]
+      ))
+    state = updateState(state, previous_block_root, some(new_block), {}).state
+
+    check:
+      state.latest_attestations.len == 1
+
+    # TODO Can't run more than 127 for now:
+    # https://github.com/ethereum/eth2.0-specs/issues/352
+    while state.slot < 127:
+      state = updateState(
+        state, previous_block_root, none(BeaconBlock), {}).state
+
+    # Would need to process more epochs for the attestation to be removed from
+    # the state! (per above bug)
+    #
+    # check:
+    #  state.latest_attestations.len == 0

tests/testutil.nim

@@ -6,9 +6,9 @@
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
 import
-  options, milagro_crypto,
+  options, milagro_crypto, sequtils,
   ../beacon_chain/[extras, ssz, state_transition],
-  ../beacon_chain/spec/[crypto, datatypes, digest, helpers]
+  ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers]
 
 const
   randaoRounds = 100
@@ -107,7 +107,7 @@ proc addBlock*(
       randao_reveal: hackReveal(proposer),
       candidate_pow_receipt_root: Eth2Digest(), # TODO
       signature: ValidatorSig(), # we need the rest of the block first!
-      body: BeaconBlockBody() # TODO throw in stuff here...
+      body: body
     )
 
   var block_ok: bool
@@ -156,3 +156,49 @@ proc makeBlock*(
   # because the block includes the state root.
   var next_state = state
   addBlock(next_state, previous_block_root, body)
+
+proc find_shard_committee(
+    sacs: openArray[ShardCommittee], validator_index: Uint24): ShardCommittee =
+  for sac in sacs:
+    if validator_index in sac.committee: return sac
+  doAssert false
+
+proc makeAttestation*(
+    state: BeaconState, beacon_block_root: Eth2Digest,
+    validator_index: Uint24): Attestation =
+
+  let new_state = updateState(
+    state, beacon_block_root, none(BeaconBlock), {skipValidation})
+  let
+    sac = find_shard_committee(
+      get_shard_committees_at_slot(state, state.slot), validator_index)
+    validator = state.validator_registry[validator_index]
+    sac_index = sac.committee.find(validator_index)
+
+    data = AttestationData(
+      slot: state.slot,
+      shard: sac.shard,
+      beacon_block_root: beacon_block_root,
+      epoch_boundary_root: Eth2Digest(), # TODO
+      shard_block_root: Eth2Digest(), # TODO
+      latest_crosslink_root: Eth2Digest(), # TODO
+      justified_slot: state.justified_slot,
+      justified_block_root:
+        get_block_root(new_state.state, state.justified_slot),
+    )
+
+  assert sac_index != -1, "find_shard_committe should guarantee this"
+
+  var
+    participation_bitfield = repeat(0'u8, ceil_div8(sac.committee.len))
+  bitSet(participation_bitfield, sac_index)
+
+  let
+    msg = hash_tree_root_final(data)
+
+  Attestation(
+    data: data,
+    participation_bitfield: participation_bitfield,
+    aggregate_signature: signMessage(
+      hackPrivKey(validator), @(msg.data) & @[0'u8])
+  )