Use O(n) algorithm in initialize_beacon_state_from_eth1; Avoid unnecessary merkle proofs generation

beacon_chain/eth1_monitor.nim

@@ -430,9 +430,9 @@ proc init*(T: type Eth1Monitor,
     dataProvider: dataProvider,
     eth1Progress: newAsyncEvent())
 
-proc allDepositsUpTo(m: Eth1Monitor, totalDeposits: uint64): seq[Deposit] =
+proc allDepositsUpTo(m: Eth1Monitor, totalDeposits: uint64): seq[DepositData] =
   for i in 0'u64 ..< totalDeposits:
-    result.add Deposit(data: m.db.deposits.get(i))
+    result.add m.db.deposits.get(i)
 
 proc createGenesisState(m: Eth1Monitor, eth1Block: Eth1Block): BeaconStateRef =
   notice "Generating genesis state",
@@ -443,7 +443,6 @@ proc createGenesisState(m: Eth1Monitor, eth1Block: Eth1Block): BeaconStateRef =
     activeValidators = eth1Block.activeValidatorsCount
 
   var deposits = m.allDepositsUpTo(eth1Block.voteData.deposit_count)
-  attachMerkleProofs deposits
 
   result = initialize_beacon_state_from_eth1(
     m.preset,

beacon_chain/interop.nim

@@ -39,15 +39,12 @@ func makeDeposit*(
     preset: RuntimePreset,
     pubkey: ValidatorPubKey, privkey: ValidatorPrivKey, epoch = 0.Epoch,
     amount: Gwei = MAX_EFFECTIVE_BALANCE.Gwei,
-    flags: UpdateFlags = {}): Deposit =
+    flags: UpdateFlags = {}): DepositData =
-  var
+  result = DepositData(
-    ret = Deposit(
+    amount: amount,
-      data: DepositData(
+    pubkey: pubkey,
-        amount: amount,
+    withdrawal_credentials: makeWithdrawalCredentials(pubkey))
-        pubkey: pubkey,
-        withdrawal_credentials: makeWithdrawalCredentials(pubkey)))
 
   if skipBLSValidation notin flags:
-    ret.data.signature = preset.get_deposit_signature(ret.data, privkey)
+    result.signature = preset.get_deposit_signature(result, privkey)
 
-  ret

beacon_chain/nimbus_beacon_node.nim

@@ -1080,11 +1080,9 @@ programMain:
              err = formatMsg(err, config.testnetDepositsFile.string)
       quit 1
 
-    var deposits: seq[Deposit]
+    var deposits: seq[DepositData]
     for i in config.firstValidator.int ..< launchPadDeposits.len:
-      deposits.add Deposit(data: launchPadDeposits[i] as DepositData)
+      deposits.add(launchPadDeposits[i] as DepositData)
-
-    attachMerkleProofs(deposits)
 
     let
       startTime = uint64(times.toUnix(times.getTime()) + config.genesisOffset)

beacon_chain/spec/beaconstate.nim

@@ -52,16 +52,16 @@ func decrease_balance*(
       state.balances[index] - delta
 
 # https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#deposits
-func get_validator_from_deposit(state: BeaconState, deposit: Deposit):
+func get_validator_from_deposit(state: BeaconState, deposit: DepositData):
     Validator =
   let
-    amount = deposit.data.amount
+    amount = deposit.amount
     effective_balance = min(
       amount - amount mod EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE)
 
   Validator(
-    pubkey: deposit.data.pubkey,
+    pubkey: deposit.pubkey,
-    withdrawal_credentials: deposit.data.withdrawal_credentials,
+    withdrawal_credentials: deposit.withdrawal_credentials,
     activation_eligibility_epoch: FAR_FUTURE_EPOCH,
     activation_epoch: FAR_FUTURE_EPOCH,
     exit_epoch: FAR_FUTURE_EPOCH,
@@ -110,7 +110,7 @@ proc process_deposit*(preset: RuntimePreset,
         return ok()
 
     # Add validator and balance entries
-    state.validators.add(get_validator_from_deposit(state, deposit))
+    state.validators.add(get_validator_from_deposit(state, deposit.data))
     state.balances.add(amount)
   else:
      # Increase balance by deposit amount
@@ -221,7 +221,7 @@ proc initialize_beacon_state_from_eth1*(
     preset: RuntimePreset,
     eth1_block_hash: Eth2Digest,
     eth1_timestamp: uint64,
-    deposits: openArray[Deposit],
+    deposits: openArray[DepositData],
     flags: UpdateFlags = {}): BeaconStateRef {.nbench.} =
   ## Get the genesis ``BeaconState``.
   ##
@@ -261,15 +261,36 @@ proc initialize_beacon_state_from_eth1*(
   # Seed RANDAO with Eth1 entropy
   state.randao_mixes.fill(eth1_block_hash)
 
-  # Process deposits
+  var merkleizer = createMerkleizer(2'i64^DEPOSIT_CONTRACT_TREE_DEPTH)
-  let
+  for i, deposit in deposits:
-    leaves = deposits.mapIt(it.data)
+    let htr = hash_tree_root(deposit)
-  var i = 0
+    merkleizer.addChunk(htr.data)
-  for prefix_root in hash_tree_roots_prefix(
+
-      leaves, 2'i64^DEPOSIT_CONTRACT_TREE_DEPTH):
+  # This is already known in the Eth1 monitor, but it would be too
-    state.eth1_data.deposit_root = prefix_root
+  # much work to refactor all the existing call sites in the test suite
-    discard process_deposit(preset, state[], deposits[i], flags)
+  state.eth1_data.deposit_root = mixInLength(merkleizer.getFinalHash(),
-    i += 1
+                                             deposits.len)
+  state.eth1_deposit_index = deposits.lenu64
+
+  var pubkeyToIndex = initTable[ValidatorPubKey, int]()
+  for idx, deposit in deposits:
+    let
+      pubkey = deposit.pubkey
+      amount = deposit.amount
+
+    pubkeyToIndex.withValue(pubkey, foundIdx) do:
+      # Increase balance by deposit amount
+      increase_balance(state[], ValidatorIndex foundIdx[], amount)
+    do:
+      if skipBlsValidation in flags or
+         verify_deposit_signature(preset, deposit):
+        state.validators.add(get_validator_from_deposit(state[], deposit))
+        state.balances.add(amount)
+        pubkeyToIndex[pubkey] = idx
+      else:
+        # Invalid deposits are perfectly possible
+        trace "Skipping deposit with invalid signature",
+          deposit = shortLog(deposit)
 
   # Process activations
   for validator_index in 0 ..< state.validators.len:
@@ -293,7 +314,7 @@ proc initialize_hashed_beacon_state_from_eth1*(
     preset: RuntimePreset,
     eth1_block_hash: Eth2Digest,
     eth1_timestamp: uint64,
-    deposits: openArray[Deposit],
+    deposits: openArray[DepositData],
     flags: UpdateFlags = {}): HashedBeaconState =
   let genesisState = initialize_beacon_state_from_eth1(
     preset, eth1_block_hash, eth1_timestamp, deposits, flags)

beacon_chain/ssz/merkleization.nim

@@ -684,13 +684,3 @@ func hash_tree_root*(x: auto): Eth2Digest {.raises: [Defect].} =
 
   trs "HASH TREE ROOT FOR ", name(type x), " = ", "0x", $result
 
-iterator hash_tree_roots_prefix*[T](lst: openArray[T], limit: static Limit): Eth2Digest =
-  # This is a particular type's instantiation of a general fold, reduce,
-  # accumulation, prefix sums, etc family of operations. As long as that
-  # Eth1 deposit case is the only notable example -- the usual uses of a
-  # list involve, at some point, tree-hashing it -- finalized hashes are
-  # the only abstraction that escapes from this module this way.
-  var merkleizer = createMerkleizer(limit)
-  for i, elem in lst:
-    merkleizer.addChunk(hash_tree_root(elem).data)
-    yield mixInLength(merkleizer.getFinalHash(), i + 1)

research/simutils.nim

@@ -66,8 +66,13 @@ proc loadGenesis*(validators: Natural, validate: bool): ref HashedBeaconState =
 
     echo "Generating Genesis..."
 
-    res.data =
+    res.data = initialize_beacon_state_from_eth1(
-      initialize_beacon_state_from_eth1(defaultRuntimePreset, Eth2Digest(), 0, deposits, flags)[]
+      defaultRuntimePreset,
+      Eth2Digest(),
+      0,
+      deposits,
+      flags)[]
+
     res.root = hash_tree_root(res.data)
 
     echo &"Saving to {fn}..."

tests/mocking/mock_deposits.nim

@@ -46,7 +46,7 @@ func mockDepositData(
   ret
 
 template mockGenesisDepositsImpl(
-        result: seq[Deposit],
+        result: seq[DepositData],
         validatorCount: uint64,
         amount: untyped,
         flags: UpdateFlags = {},
@@ -65,7 +65,7 @@ template mockGenesisDepositsImpl(
       updateAmount
 
       # DepositData
-      result[valIdx].data = mockDepositData(MockPubKeys[valIdx], amount)
+      result[valIdx] = mockDepositData(MockPubKeys[valIdx], amount)
   else: # With signing
     var depositsDataHash: seq[Eth2Digest]
     var depositsData: seq[DepositData]
@@ -78,17 +78,17 @@ template mockGenesisDepositsImpl(
       updateAmount
 
       # DepositData
-      result[valIdx].data = mockDepositData(
+      result[valIdx] = mockDepositData(
         MockPubKeys[valIdx], MockPrivKeys[valIdx], amount, flags)
 
-      depositsData.add result[valIdx].data
+      depositsData.add result[valIdx]
-      depositsDataHash.add hash_tree_root(result[valIdx].data)
+      depositsDataHash.add hash_tree_root(result[valIdx])
 
 proc mockGenesisBalancedDeposits*(
         validatorCount: uint64,
         amountInEth: Positive,
         flags: UpdateFlags = {}
-      ): seq[Deposit] =
+      ): seq[DepositData] =
   ## The amount should be strictly positive
   ## - 1 is the minimum deposit amount (MIN_DEPOSIT_AMOUNT)
   ## - 16 is the ejection balance (EJECTION_BALANCE)
@@ -100,28 +100,6 @@ proc mockGenesisBalancedDeposits*(
   let amount = amountInEth.uint64 * 10'u64^9
   mockGenesisDepositsImpl(result, validatorCount,amount,flags):
     discard
-  attachMerkleProofs(result)
-
-proc mockGenesisUnBalancedDeposits*(
-        validatorCount: uint64,
-        amountRangeInEth: Slice[int], # TODO: use "Positive", Nim range bug
-        flags: UpdateFlags = {}
-      ): seq[Deposit] =
-
-  ## The range of deposit amount should be strictly positive
-  ## - 1 is the minimum deposit amount (MIN_DEPOSIT_AMOUNT)
-  ## - 16 is the ejection balance (EJECTION_BALANCE)
-  ## - 32 is the max effective balance (MAX_EFFECTIVE_BALANCE)
-  ##   ETH beyond do not contribute more for staking.
-  ##
-  ## Only validators with 32 ETH will be active at genesis
-
-  var rng {.global.} = initRand(0x42) # Fixed seed for reproducibility
-  var amount: uint64
-
-  mockGenesisDepositsImpl(result, validatorCount, amount, flags):
-    amount = rng.rand(amountRangeInEth).uint64 * 10'u64^9
-  attachMerkleProofs(result)
 
 proc mockUpdateStateForNewDeposit*(
        state: var BeaconState,

tests/test_interop.nim

@@ -141,12 +141,11 @@ suiteReport "Interop":
     # Check against https://github.com/protolambda/zcli:
     # zcli keys generate --to 64 | zcli genesis mock --genesis-time 1570500000 > /tmp/state.ssz
     # zcli hash-tree-root state /tmp/state.ssz
-    var deposits: seq[Deposit]
+    var deposits: seq[DepositData]
 
     for i in 0..<64:
       let privKey = makeInteropPrivKey(i)
       deposits.add makeDeposit(defaultRuntimePreset, privKey.toPubKey(), privKey)
-    attachMerkleProofs(deposits)
 
     const genesis_time = 1570500000
     var

tests/test_ssz_merkleization.nim

@@ -215,6 +215,18 @@ for prelude in [0, 1, 2, 5, 6, 12, 13, 16]:
       testMultiProofsGeneration(prelude, proofs, followUpHashes, 128)
       testMultiProofsGeneration(prelude, proofs, followUpHashes, 5000)
 
+iterator hash_tree_roots_prefix[T](lst: openArray[T],
+                                   limit: static Limit): Eth2Digest =
+  # This is a particular type's instantiation of a general fold, reduce,
+  # accumulation, prefix sums, etc family of operations. As long as that
+  # Eth1 deposit case is the only notable example -- the usual uses of a
+  # list involve, at some point, tree-hashing it -- finalized hashes are
+  # the only abstraction that escapes from this module this way.
+  var merkleizer = createMerkleizer(limit)
+  for i, elem in lst:
+    merkleizer.addChunk(hash_tree_root(elem).data)
+    yield mixInLength(merkleizer.getFinalHash(), i + 1)
+
 func attachMerkleProofsReferenceImpl(deposits: var openArray[Deposit]) =
   let
     deposit_data_roots = mapIt(deposits, it.data.hash_tree_root)

tests/testblockutil.nim

@@ -34,7 +34,7 @@ func hackPrivKey*(v: Validator): ValidatorPrivKey =
   let i = int(uint64.fromBytesLE(bytes))
   makeFakeValidatorPrivKey(i)
 
-func makeDeposit(i: int, flags: UpdateFlags): Deposit =
+func makeDeposit(i: int, flags: UpdateFlags): DepositData =
   ## Ugly hack for now: we stick the private key in withdrawal_credentials
   ## which means we can repro private key and randao reveal from this data,
   ## for testing :)
@@ -43,31 +43,20 @@ func makeDeposit(i: int, flags: UpdateFlags): Deposit =
     pubkey = privkey.toPubKey()
     withdrawal_credentials = makeFakeHash(i)
 
-  result = Deposit(
+  result = DepositData(
-    data: DepositData(
+    pubkey: pubkey,
-      pubkey: pubkey,
+    withdrawal_credentials: withdrawal_credentials,
-      withdrawal_credentials: withdrawal_credentials,
+    amount: MAX_EFFECTIVE_BALANCE)
-      amount: MAX_EFFECTIVE_BALANCE,
-    )
-  )
 
   if skipBLSValidation notin flags:
-    result.data.signature = get_deposit_signature(
+    result.signature = get_deposit_signature(
-      defaultRuntimePreset, result.data, privkey)
+      defaultRuntimePreset, result, privkey)
 
 proc makeInitialDeposits*(
-    n = SLOTS_PER_EPOCH, flags: UpdateFlags = {}): seq[Deposit] =
+    n = SLOTS_PER_EPOCH, flags: UpdateFlags = {}): seq[DepositData] =
   for i in 0..<n.int:
     result.add makeDeposit(i, flags)
 
-  # This needs to be done as a batch, since the Merkle proof of the i'th
-  # deposit depends on the deposit (data) of the 0th through (i-1)st, of
-  # deposits. Computing partial hash_tree_root sequences of DepositData,
-  # and ideally (but not yet) efficiently only once calculating a Merkle
-  # tree utilizing as much of the shared substructure as feasible, means
-  # attaching proofs all together, as a separate step.
-  attachMerkleProofs(result)
-
 func signBlock*(
     fork: Fork, genesis_validators_root: Eth2Digest, blck: BeaconBlock,
     privKey: ValidatorPrivKey, flags: UpdateFlags = {}): SignedBeaconBlock =

tests/testutil.nim

@@ -104,9 +104,11 @@ proc makeTestDB*(tailState: BeaconState, tailBlock: SignedBeaconBlock): BeaconCh
 proc makeTestDB*(validators: Natural): BeaconChainDB =
   let
     genState = initialize_beacon_state_from_eth1(
-      defaultRuntimePreset, Eth2Digest(), 0,
+      defaultRuntimePreset,
+      Eth2Digest(),
+      0,
       makeInitialDeposits(validators.uint64, flags = {skipBlsValidation}),
-        {skipBlsValidation})
+      {skipBlsValidation})
     genBlock = get_initial_beacon_block(genState[])
   makeTestDB(genState[], genBlock)