re-enable test_interop based on zcli with 0.9.1 specs and update initialize_beacon_state_from_eth1(...) to 0.9.1 (#569)

* re-enable test_interop based on zcli with 0.9.1 specs and update initialize_beacon_state_from_eth1(...) to 0.9.1

* switch many procs to funcs

* fix import os.nim instead; ospaths is deprecated [Deprecated] warnings

README.md

@@ -25,7 +25,7 @@ This prevents LFS during unusual clones (i.e. when you add `--recurse-submodules
 ## Related
 
 * [status-im/nimbus](https://github.com/status-im/nimbus/): main Nimbus repository - start here to learn more about the Nimbus eco-system
-* [ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs/blob/master/specs/core/0_beacon-chain.md): Serenity specification that this project implements
+* [ethereum/eth2.0-specs](https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/specs/core/0_beacon-chain.md): Serenity specification that this project implements
 * [ethereum/beacon\_chain](https://github.com/ethereum/beacon_chain): reference implementation from the Ethereum foundation
 
 You can check where the beacon chain fits in the Ethereum research ecosystem in the [Status Athenaeum](https://github.com/status-im/athenaeum/blob/b465626cc551e361492e56d32517b2cdadd7493f/ethereum_research_records.json#L38).

beacon_chain/attestation_pool.nim

@@ -7,7 +7,7 @@ import
 
 logScope: topics = "attpool"
 
-proc init*(T: type AttestationPool, blockPool: BlockPool): T =
+func init*(T: type AttestationPool, blockPool: BlockPool): T =
   # TODO blockPool is only used when resolving orphaned attestations - it should
   #      probably be removed as a dependency of AttestationPool (or some other
   #      smart refactoring)
@@ -129,7 +129,7 @@ proc slotIndex(
 
   int(attestationSlot - pool.startingSlot)
 
-proc updateLatestVotes(
+func updateLatestVotes(
     pool: var AttestationPool, state: BeaconState, attestationSlot: Slot,
     participants: seq[ValidatorIndex], blck: BlockRef) =
   for validator in participants:
@@ -245,7 +245,7 @@ proc add*(pool: var AttestationPool,
       validations = 1,
       cat = "filtering"
 
-proc addUnresolved*(pool: var AttestationPool, attestation: Attestation) =
+func addUnresolved*(pool: var AttestationPool, attestation: Attestation) =
   pool.unresolved[attestation.data.beacon_block_root] =
     UnresolvedAttestation(
       attestation: attestation,
@@ -333,15 +333,6 @@ proc getAttestationsForBlock*(
     if result.len >= MAX_ATTESTATIONS:
       return
 
-proc getAttestationsForTargetEpoch*(
-  pool: AttestationPool, state: var BeaconState,
-    epoch: Epoch): seq[Attestation] =
-  # TODO quick testing kludge
-  let begin_slot = compute_start_slot_at_epoch(epoch).uint64
-  let end_slot_minus1 = (compute_start_slot_at_epoch(epoch+1) - 1).uint64
-  for s in begin_slot .. end_slot_minus1:
-    result.add getAttestationsForBlock(pool, state, s.Slot)
-
 proc resolve*(pool: var AttestationPool, cache: var StateData) =
   var
     done: seq[Eth2Digest]
@@ -365,6 +356,6 @@ proc resolve*(pool: var AttestationPool, cache: var StateData) =
 
     pool.add(cache.data.data, a.blck, a.attestation)
 
-proc latestAttestation*(
+func latestAttestation*(
     pool: AttestationPool, pubKey: ValidatorPubKey): BlockRef =
   pool.latestAttestations.getOrDefault(pubKey)

beacon_chain/beacon_node.nim

@@ -259,7 +259,7 @@ proc addLocalValidators(node: BeaconNode, state: BeaconState) =
 
   info "Local validators attached ", count = node.attachedValidators.count
 
-proc getAttachedValidator(node: BeaconNode,
+func getAttachedValidator(node: BeaconNode,
                           state: BeaconState,
                           idx: ValidatorIndex): AttachedValidator =
   let validatorKey = state.validators[idx].pubkey

beacon_chain/block_pool.nim

@@ -8,13 +8,13 @@ declareCounter beacon_reorgs_total, "Total occurrences of reorganizations of the
 
 logScope: topics = "blkpool"
 
-proc parent*(bs: BlockSlot): BlockSlot =
+func parent*(bs: BlockSlot): BlockSlot =
   BlockSlot(
     blck: if bs.slot > bs.blck.slot: bs.blck else: bs.blck.parent,
     slot: bs.slot - 1
   )
 
-proc link(parent, child: BlockRef) =
+func link(parent, child: BlockRef) =
   doAssert (not (parent.root == Eth2Digest() or child.root == Eth2Digest())),
     "blocks missing root!"
   doAssert parent.root != child.root, "self-references not allowed"
@@ -22,16 +22,16 @@ proc link(parent, child: BlockRef) =
   child.parent = parent
   parent.children.add(child)
 
-proc init*(T: type BlockRef, root: Eth2Digest, slot: Slot): BlockRef =
+func init*(T: type BlockRef, root: Eth2Digest, slot: Slot): BlockRef =
   BlockRef(
     root: root,
     slot: slot
   )
 
-proc init*(T: type BlockRef, root: Eth2Digest, blck: BeaconBlock): BlockRef =
+func init*(T: type BlockRef, root: Eth2Digest, blck: BeaconBlock): BlockRef =
   BlockRef.init(root, blck.slot)
 
-proc findAncestorBySlot*(blck: BlockRef, slot: Slot): BlockSlot =
+func findAncestorBySlot*(blck: BlockRef, slot: Slot): BlockSlot =
   ## Find the first ancestor that has a slot number less than or equal to `slot`
   assert(not blck.isNil)
   var ret = blck
@@ -132,7 +132,7 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool =
     heads: @[head]
   )
 
-proc addSlotMapping(pool: BlockPool, slot: uint64, br: BlockRef) =
+func addSlotMapping(pool: BlockPool, slot: uint64, br: BlockRef) =
   proc addIfMissing(s: var seq[BlockRef], v: BlockRef) =
     if v notin s:
       s.add(v)
@@ -306,11 +306,11 @@ proc add*(
           (parentSlot.uint64 mod SLOTS_PER_EPOCH.uint64))
   )
 
-proc getRef*(pool: BlockPool, root: Eth2Digest): BlockRef =
+func getRef*(pool: BlockPool, root: Eth2Digest): BlockRef =
   ## Retrieve a resolved block reference, if available
   pool.blocks.getOrDefault(root)
 
-proc getBlockRange*(pool: BlockPool, headBlock: Eth2Digest,
+func getBlockRange*(pool: BlockPool, headBlock: Eth2Digest,
                     startSlot: Slot, skipStep: Natural,
                     output: var openarray[BlockRef]): Natural =
   ## This function populates an `output` buffer of blocks
@@ -385,7 +385,7 @@ proc get*(pool: BlockPool, root: Eth2Digest): Option[BlockData] =
   else:
     none(BlockData)
 
-proc getOrResolve*(pool: var BlockPool, root: Eth2Digest): BlockRef =
+func getOrResolve*(pool: var BlockPool, root: Eth2Digest): BlockRef =
   ## Fetch a block ref, or nil if not found (will be added to list of
   ## blocks-to-resolve)
   result = pool.getRef(root)
@@ -397,7 +397,7 @@ iterator blockRootsForSlot*(pool: BlockPool, slot: uint64|Slot): Eth2Digest =
   for br in pool.blocksBySlot.getOrDefault(slot.uint64, @[]):
     yield br.root
 
-proc checkMissing*(pool: var BlockPool): seq[FetchRecord] =
+func checkMissing*(pool: var BlockPool): seq[FetchRecord] =
   ## Return a list of blocks that we should try to resolve from other client -
   ## to be called periodically but not too often (once per slot?)
   var done: seq[Eth2Digest]
@@ -678,7 +678,7 @@ proc updateHead*(pool: BlockPool, state: var StateData, blck: BlockRef) =
           not pool.heads[n].blck.isAncestorOf(pool.finalizedHead.blck):
         pool.heads.del(n)
 
-proc latestJustifiedBlock*(pool: BlockPool): BlockSlot =
+func latestJustifiedBlock*(pool: BlockPool): BlockSlot =
   ## Return the most recent block that is justified and at least as recent
   ## as the latest finalized block
 

beacon_chain/fork_choice.nim

@@ -4,7 +4,7 @@ import
   ./spec/[datatypes, crypto, helpers],
   ./attestation_pool, ./beacon_node_types, ./ssz
 
-proc get_ancestor(blck: BlockRef, slot: Slot): BlockRef =
+func get_ancestor(blck: BlockRef, slot: Slot): BlockRef =
   if blck.slot == slot:
     blck
   elif blck.slot < slot:
@@ -16,7 +16,7 @@ proc get_ancestor(blck: BlockRef, slot: Slot): BlockRef =
 # The structure of this code differs from the spec since we use a different
 # strategy for storing states and justification points - it should nonetheless
 # be close in terms of functionality.
-proc lmdGhost*(
+func lmdGhost*(
     pool: AttestationPool, start_state: BeaconState,
     start_block: BlockRef): BlockRef =
   # TODO: a Fenwick Tree datastructure to keep track of cumulated votes

beacon_chain/interop.nim

@@ -1,5 +1,5 @@
 import
-  ospaths,
+  os,
   stew/endians2, stint,
   ./extras, ./ssz,
   spec/[crypto, datatypes, digest, helpers]

beacon_chain/spec/beaconstate.nim

@@ -174,7 +174,7 @@ func slash_validator*(state: var BeaconState, slashed_index: ValidatorIndex,
   increase_balance(
     state, whistleblower_index, whistleblowing_reward - proposer_reward)
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#genesis
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/specs/core/0_beacon-chain.md#genesis
 func initialize_beacon_state_from_eth1*(
     eth1_block_hash: Eth2Digest,
     eth1_timestamp: uint64,
@@ -213,6 +213,9 @@ func initialize_beacon_state_from_eth1*(
       )
   )
 
+  # Seed RANDAO with Eth1 entropy
+  state.randao_mixes.fill(eth1_block_hash)
+
   # Process deposits
   let leaves = deposits.mapIt(it.data)
   for i, deposit in deposits:
@@ -237,7 +240,7 @@ func initialize_beacon_state_from_eth1*(
 
   state
 
-proc is_valid_genesis_state*(state: BeaconState): bool =
+func is_valid_genesis_state*(state: BeaconState): bool =
   if state.genesis_time < MIN_GENESIS_TIME:
     return false
   if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT:
@@ -486,7 +489,7 @@ proc process_attestation*(
   else:
     false
 
-proc makeAttestationData*(
+func makeAttestationData*(
     state: BeaconState, slot: Slot, committee_index: uint64,
     beacon_block_root: Eth2Digest): AttestationData =
   ## Create an attestation / vote for the block `beacon_block_root` using the

beacon_chain/spec/crypto.nim

@@ -122,7 +122,7 @@ func shortLog*(x: BlsValue): string =
 func shortLog*(x: BlsCurveType): string =
   ($x)[0..7]
 
-proc hash*(x: BlsValue): Hash {.inline.} =
+func hash*(x: BlsValue): Hash {.inline.} =
   if x.kind == Real:
     hash x.blsValue.getBytes()
   else:
@@ -145,19 +145,19 @@ func pubKey*(pk: ValidatorPrivKey): ValidatorPubKey =
   else:
     pk.getKey
 
-proc init(T: type VerKey): VerKey =
+func init(T: type VerKey): VerKey =
   result.point.inf()
 
-proc init(T: type SigKey): SigKey =
+func init(T: type SigKey): SigKey =
   result.point.inf()
 
-proc combine*[T](values: openarray[BlsValue[T]]): BlsValue[T] =
+func combine*[T](values: openarray[BlsValue[T]]): BlsValue[T] =
   result = BlsValue[T](kind: Real, blsValue: T.init())
 
   for value in values:
     result.blsValue.combine(value.blsValue)
 
-proc combine*[T](x: var BlsValue[T], other: BlsValue[T]) =
+func combine*[T](x: var BlsValue[T], other: BlsValue[T]) =
   doAssert x.kind == Real and other.kind == Real
   x.blsValue.combine(other.blsValue)
 

beacon_chain/spec/digest.nim

@@ -50,7 +50,7 @@ func eth2hash*(v: openArray[byte]): Eth2Digest {.inline.} =
   ctx.update(v)
   ctx.finish()
 
-proc update*(ctx: var Sha2Context; digest: Eth2Digest) =
+func update*(ctx: var Sha2Context; digest: Eth2Digest) =
   ctx.update digest.data
 
 template withEth2Hash*(body: untyped): Eth2Digest =

beacon_chain/spec/state_transition_epoch.nim

@@ -366,7 +366,7 @@ func process_slashings*(state: var BeaconState) =
       decrease_balance(state, index.ValidatorIndex, penalty)
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/specs/core/0_beacon-chain.md#final-updates
-proc process_final_updates*(state: var BeaconState) =
+func process_final_updates*(state: var BeaconState) =
   let
     current_epoch = get_current_epoch(state)
     next_epoch = current_epoch + 1

beacon_chain/spec/state_transition_helpers.nim

@@ -22,17 +22,17 @@ func shortLog*(x: Checkpoint): string =
 # Helpers used in epoch transition and trace-level block transition
 # --------------------------------------------------------
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#get_attesting_indices
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/specs/core/0_beacon-chain.md#helper-functions-1
-# TODO there's another one of these, check for redundancy
 func get_attesting_indices*(
     state: BeaconState, attestations: openarray[PendingAttestation],
     stateCache: var StateCache): HashSet[ValidatorIndex] =
+  # This is part of get_unslashed_attesting_indices(...) in spec.
   result = initHashSet[ValidatorIndex]()
   for a in attestations:
     result = result.union(get_attesting_indices(
       state, a.data, a.aggregation_bits, stateCache))
 
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#helper-functions-1
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/specs/core/0_beacon-chain.md#helper-functions-1
 func get_unslashed_attesting_indices*(
     state: BeaconState, attestations: openarray[PendingAttestation],
     stateCache: var StateCache): HashSet[ValidatorIndex] =

beacon_chain/statusbar.nim

@@ -49,7 +49,7 @@ func loadLayout(layout: string): Layout {.raises: [Defect, ValueError].} =
   result.cellsLeft = loadCellsLayout(sections[0])
   if sections.len == 2: result.cellsRight = loadCellsLayout(sections[1])
 
-proc updateContent(cell: var StatusBarCell, model: DataItemResolver) =
+func updateContent(cell: var StatusBarCell, model: DataItemResolver) =
   cell.content.setLen 0
   for fragment in cell.contentFragments:
     case fragment[0]
@@ -58,11 +58,11 @@ proc updateContent(cell: var StatusBarCell, model: DataItemResolver) =
     of ikExpr, ikVar:
       cell.content.add model(fragment[1])
 
-proc updateCells(cells: var seq[StatusBarCell], model: DataItemResolver) =
+func updateCells(cells: var seq[StatusBarCell], model: DataItemResolver) =
   for cell in mitems(cells):
     cell.updateContent(model)
 
-proc update*(s: var StatusBarView) =
+func update*(s: var StatusBarView) =
   updateCells s.layout.cellsLeft, s.model
   updateCells s.layout.cellsRight, s.model
 

beacon_chain/time.nim

@@ -60,6 +60,8 @@ func toSlot*(c: BeaconClock, t: Time): tuple[afterGenesis: bool, slot: Slot] =
 func toBeaconTime*(s: Slot, offset = chronos.seconds(0)): BeaconTime =
   BeaconTime(int64(uint64(s) * SECONDS_PER_SLOT) + seconds(offset))
 
+# TODO on Travis ARM64 CIs, this claims to have side effects, but neither Linux
+# nor Mac OS x86 CIs exhibit this behavior.
 proc now*(c: BeaconClock): BeaconTime =
   ## Current time, in slots - this may end up being less than GENESIS_SLOT(!)
   toBeaconTime(c, getTime())
@@ -75,7 +77,7 @@ proc fromNow*(c: BeaconClock, t: BeaconTime): tuple[inFuture: bool, offset: Dura
 proc fromNow*(c: BeaconClock, slot: Slot): tuple[inFuture: bool, offset: Duration] =
   c.fromNow(slot.toBeaconTime())
 
-proc saturate*(d: tuple[inFuture: bool, offset: Duration]): Duration =
+func saturate*(d: tuple[inFuture: bool, offset: Duration]): Duration =
   if d.inFuture: d.offset else: seconds(0)
 
 proc addTimer*(fromNow: Duration, cb: CallbackFunc, udata: pointer = nil) =

beacon_chain/trusted_state_snapshots.nim

@@ -1,5 +1,5 @@
 import
-  ospaths, chronos, json_serialization,
+  os, chronos, json_serialization,
   spec/[datatypes], beacon_chain_db
 
 const

beacon_chain/validator_keygen.nim

@@ -1,5 +1,5 @@
 import
-  os, ospaths, strutils,
+  os, strutils,
   chronicles, chronos, blscurve, nimcrypto, json_serialization, serialization,
   web3, stint, eth/keys,
   spec/[datatypes, digest, crypto], conf, ssz, interop

beacon_chain/validator_pool.nim

@@ -4,7 +4,7 @@ import
   spec/[datatypes, crypto, digest, helpers], ssz,
   beacon_node_types
 
-proc init*(T: type ValidatorPool): T =
+func init*(T: type ValidatorPool): T =
   result.validators = initTable[ValidatorPubKey, AttachedValidator]()
 
 template count*(pool: ValidatorPool): int =
@@ -22,7 +22,7 @@ proc addLocalValidator*(pool: var ValidatorPool,
 
   info "Local validator attached", pubKey, validator = shortLog(v)
 
-proc getValidator*(pool: ValidatorPool,
+func getValidator*(pool: ValidatorPool,
                    validatorKey: ValidatorPubKey): AttachedValidator =
   pool.validators.getOrDefault(validatorKey)
 

tests/mocking/mock_attestations.nim

@@ -11,7 +11,6 @@
 import
   # Standard library
   sets,
-  # 0.19.6 shims
   # Specs
   ../../beacon_chain/spec/[datatypes, beaconstate, helpers, validator, crypto],
   # Internals

tests/mocking/mock_deposits.nim

@@ -11,8 +11,6 @@
 import
   # Standard library
   math, random,
-  # 0.19.6 shims
-  stew/objects, # import default
   # Specs
   ../../beacon_chain/spec/[datatypes, crypto, helpers, digest, beaconstate],
   # Internals

tests/simulation/run_node.sh

@@ -23,15 +23,15 @@ if [ "${NAT:-}" == "1" ]; then
   NAT_FLAG="--nat:any"
 fi
 
-mkdir -p $DATA_DIR/validators
+mkdir -p "$DATA_DIR/validators"
 rm -f $DATA_DIR/validators/*
 
 if [[ $NODE_ID -lt $TOTAL_NODES ]]; then
   FIRST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / TOTAL_NODES) * NODE_ID ))
   LAST_VALIDATOR_IDX=$(( (NUM_VALIDATORS / TOTAL_NODES) * (NODE_ID + 1) - 1 ))
 
-  pushd $VALIDATORS_DIR >/dev/null
+  pushd "$VALIDATORS_DIR" >/dev/null
-    cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) $DATA_DIR/validators
+    cp $(seq -s " " -f v%07g.privkey $FIRST_VALIDATOR_IDX $LAST_VALIDATOR_IDX) "$DATA_DIR/validators"
   popd >/dev/null
 fi
 

tests/spec_block_processing/test_genesis.nim

@@ -7,7 +7,7 @@
 
 
 # initialize_beacon_state_from_eth1 (beaconstate.nim)
-# https://github.com/ethereum/eth2.0-specs/blob/v0.8.1/specs/core/0_beacon-chain.md#genesis
+# https://github.com/ethereum/eth2.0-specs/blob/v0.9.1/specs/core/0_beacon-chain.md#genesis
 # ---------------------------------------------------------------
 
 {.used.}

tests/test_interop.nim

@@ -157,11 +157,11 @@ suite "Interop":
 
     let expected =
       when const_preset == "minimal":
-        "029836dbceb95c20b101f8f44470604c0912e96949aaf1dd9ad41effd92abcbf"
+        "75016055f843b92972d647a849168e8c5f559e8d41e05f94fc3f6a9665d1cabb"
       elif const_preset == "mainnet":
-        "9cd22b0ea2ec836fef591d259f0d4273669cba4c82df32cf0aa55c388ff7e432"
+        "27e4b5dfc67b97fd7d441c60bd5c92851fc1ceebe22435903183d915b3e4e678"
       else:
         "unimplemented"
     check:
-      # hash_tree_root(initialState).data.toHex() == expected
+      hash_tree_root(initialState).data.toHex() == expected
       true