Migrating the deposit contract snapshot can no longer fail on start-up (#4438)

The missing piece of data that had to be obtained previously from
the configured EL client is now part of the network metadata baked
into the binary.

beacon_chain/eth1/eth1_monitor.nim

@@ -1064,18 +1064,33 @@ template getOrDefault[T, E](r: Result[T, E]): T =
   type TT = T
   get(r, default(TT))
 
-proc init*(T: type Eth1Chain, cfg: RuntimeConfig, db: BeaconChainDB): T =
+proc init*(T: type Eth1Chain,
+           cfg: RuntimeConfig,
+           db: BeaconChainDB,
+           depositContractBlockNumber: uint64,
+           depositContractBlockHash: Eth2Digest): T =
   let
-    finalizedDeposits =
+    (finalizedBlockHash, depositContractState) =
       if db != nil:
-        db.getDepositTreeSnapshot().getOrDefault()
+        let treeSnapshot = db.getDepositTreeSnapshot()
+        if treeSnapshot.isSome:
+          (treeSnapshot.get.eth1Block, treeSnapshot.get.depositContractState)
+        else:
+          let oldSnapshot = db.getUpgradableDepositSnapshot()
+          if oldSnapshot.isSome:
+            (oldSnapshot.get.eth1Block, oldSnapshot.get.depositContractState)
+          else:
+            db.putDepositTreeSnapshot DepositTreeSnapshot(
+              eth1Block: depositContractBlockHash,
+              blockHeight: depositContractBlockNumber)
+            (depositContractBlockHash, default(DepositContractState))
       else:
-        default(DepositTreeSnapshot)
-    m = DepositsMerkleizer.init(finalizedDeposits.depositContractState)
+        (depositContractBlockHash, default(DepositContractState))
+    m = DepositsMerkleizer.init(depositContractState)
 
   T(db: db,
     cfg: cfg,
-    finalizedBlockHash: finalizedDeposits.eth1Block,
+    finalizedBlockHash: finalizedBlockHash,
     finalizedDepositsMerkleizer: m,
     headMerkleizer: copy m)
 
@@ -1095,7 +1110,8 @@ proc currentEpoch(m: Eth1Monitor): Epoch =
 
 proc init*(T: type Eth1Monitor,
            cfg: RuntimeConfig,
-           depositContractDeployedAt: BlockHashOrNumber,
+           depositContractBlockNumber: uint64,
+           depositContractBlockHash: Eth2Digest,
            db: BeaconChainDB,
            getBeaconTime: GetBeaconTimeFn,
            web3Urls: seq[string],
@@ -1108,10 +1124,15 @@ proc init*(T: type Eth1Monitor,
   for url in mitems(web3Urls):
     fixupWeb3Urls url
 
+  let eth1Chain = Eth1Chain.init(
+    cfg, db, depositContractBlockNumber, depositContractBlockHash)
+
   T(state: Initialized,
-    depositsChain: Eth1Chain.init(cfg, db),
+    depositsChain: eth1Chain,
     depositContractAddress: cfg.DEPOSIT_CONTRACT_ADDRESS,
-    depositContractDeployedAt: depositContractDeployedAt,
+    depositContractDeployedAt: BlockHashOrNumber(
+      isHash: true,
+      hash: depositContractBlockHash),
     getBeaconTime: getBeaconTime,
     web3Urls: web3Urls,
     eth1Network: eth1Network,
@@ -1121,37 +1142,6 @@ proc init*(T: type Eth1Monitor,
     blocksPerLogsRequest: targetBlocksPerLogsRequest,
     ttdReachedField: ttdReached)
 
-proc runDbMigrations*(m: Eth1Monitor) {.async.} =
-  template db: auto = m.depositsChain.db
-
-  if db.hasDepositTreeSnapshot():
-    return
-
-  # There might be an old deposit snapshot in the database that needs upgrade.
-  let oldSnapshot = db.getUpgradableDepositSnapshot()
-  if oldSnapshot.isSome:
-    let
-      hash = oldSnapshot.get.eth1Block.asBlockHash()
-      blk = awaitWithRetries m.dataProvider.getBlockByHash(hash)
-      blockNumber = uint64(blk.number)
-
-    db.putDepositTreeSnapshot oldSnapshot.get.toDepositTreeSnapshot(blockNumber)
-  elif not m.depositContractAddress.isZeroMemory:
-    # If there is no DCS record at all, create one pointing to the deployment block
-    # of the deposit contract and insert it as a starting point.
-    let blk = try:
-      awaitWithRetries m.dataProvider.getBlock(m.depositContractDeployedAt)
-    except CatchableError as e:
-      fatal "Failed to fetch deployment block",
-            depositContract = m.depositContractAddress,
-            deploymentBlock = $m.depositContractDeployedAt,
-            err = e.msg
-      quit 1
-    doAssert blk != nil, "getBlock should not return nil"
-    db.putDepositTreeSnapshot DepositTreeSnapshot(
-      eth1Block: blk.hash.asEth2Digest,
-      blockHeight: uint64 blk.number)
-
 proc safeCancel(fut: var Future[void]) =
   if not fut.isNil and not fut.finished:
     fut.cancel()
@@ -1483,8 +1473,6 @@ proc startEth1Syncing(m: Eth1Monitor, delayBeforeStart: Duration) {.async.} =
   await m.ensureDataProvider()
   doAssert m.dataProvider != nil, "close not called concurrently"
 
-  await m.runDbMigrations()
-
   # We might need to reset the chain if the new provider disagrees
   # with the previous one regarding the history of the chain or if
   # we have detected a conensus violation - our view disagreeing with

beacon_chain/networking/network_metadata.nim

@@ -12,8 +12,7 @@ else:
 
 import
   std/[sequtils, strutils, os],
-  stew/byteutils, stew/shims/macros, nimcrypto/hash,
-  eth/common/eth_types as commonEthTypes,
+  stew/[byteutils, objects], stew/shims/macros, nimcrypto/hash,
   web3/[ethtypes, conversions],
   chronicles,
   eth/common/eth_types_json_serialization,
@@ -59,7 +58,8 @@ type
       # Parsing `enr.Records` is still not possible at compile-time
       bootstrapNodes*: seq[string]
 
-      depositContractDeployedAt*: BlockHashOrNumber
+      depositContractBlock*: uint64
+      depositContractBlockHash*: Eth2Digest
 
       # Please note that we are using `string` here because SSZ.decode
       # is not currently usable at compile time and we want to load the
@@ -112,6 +112,7 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et
       configPath = path & "/config.yaml"
       deployBlockPath = path & "/deploy_block.txt"
       depositContractBlockPath = path & "/deposit_contract_block.txt"
+      depositContractBlockHashPath = path & "/deposit_contract_block_hash.txt"
       bootstrapNodesPath = path & "/bootstrap_nodes.txt"
       bootEnrPath = path & "/boot_enr.yaml"
       runtimeConfig = if fileExists(configPath):
@@ -126,22 +127,43 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et
       else:
         defaultRuntimeConfig
 
-      depositContractBlock = if fileExists(depositContractBlockPath):
+      depositContractBlockStr = if fileExists(depositContractBlockPath):
         readFile(depositContractBlockPath).strip
       else:
         ""
 
-      deployBlock = if fileExists(deployBlockPath):
+      depositContractBlockHashStr = if fileExists(depositContractBlockHashPath):
+        readFile(depositContractBlockHashPath).strip
+      else:
+        ""
+
+      deployBlockStr = if fileExists(deployBlockPath):
         readFile(deployBlockPath).strip
       else:
         ""
 
-      depositContractDeployedAt = if depositContractBlock.len > 0:
-        BlockHashOrNumber.init(depositContractBlock)
-      elif deployBlock.len > 0:
-        BlockHashOrNumber.init(deployBlock)
+      depositContractBlock = if depositContractBlockStr.len > 0:
+        parseBiggestUInt depositContractBlockStr
+      elif deployBlockStr.len > 0:
+        parseBiggestUInt deployBlockStr
+      elif not runtimeConfig.DEPOSIT_CONTRACT_ADDRESS.isDefaultValue:
+        raise newException(ValueError,
+          "A network with deposit contract should specify the " &
+          "deposit contract deployment block in a file named " &
+          "deposit_contract_block.txt or deploy_block.txt")
       else:
-        BlockHashOrNumber(isHash: false, number: 1)
+        1'u64
+
+      depositContractBlockHash = if depositContractBlockHashStr.len > 0:
+        Eth2Digest.strictParse(depositContractBlockHashStr)
+      elif (not runtimeConfig.DEPOSIT_CONTRACT_ADDRESS.isDefaultValue) and
+            depositContractBlock != 0:
+        raise newException(ValueError,
+          "A network with deposit contract should specify the " &
+          "deposit contract deployment block hash in a file " &
+          "name deposit_contract_block_hash.txt")
+      else:
+        default(Eth2Digest)
 
       bootstrapNodes = deduplicate(
         readBootstrapNodes(bootstrapNodesPath) &
@@ -162,7 +184,8 @@ proc loadEth2NetworkMetadata*(path: string, eth1Network = none(Eth1Network)): Et
       eth1Network: eth1Network,
       cfg: runtimeConfig,
       bootstrapNodes: bootstrapNodes,
-      depositContractDeployedAt: depositContractDeployedAt,
+      depositContractBlock: depositContractBlock,
+      depositContractBlockHash: depositContractBlockHash,
       genesisData: genesisData,
       genesisDepositsSnapshot: genesisDepositsSnapshot)
 
@@ -262,7 +285,6 @@ proc getMetadataForNetwork*(
 
   metadata
 
-
 proc getRuntimeConfig*(
     eth2Network: Option[string]): RuntimeConfig {.raises: [Defect, IOError].} =
   ## Returns the run-time config for a network specified on the command line

beacon_chain/nimbus_beacon_node.nim

@@ -477,7 +477,8 @@ proc init*(T: type BeaconNode,
         #      that would do only this - see Paul's proposal for this.
         let eth1Monitor = Eth1Monitor.init(
           cfg,
-          metadata.depositContractDeployedAt,
+          metadata.depositContractBlock,
+          metadata.depositContractBlockHash,
           db,
           nil,
           config.web3Urls,
@@ -573,7 +574,8 @@ proc init*(T: type BeaconNode,
   if eth1Monitor.isNil and config.web3Urls.len > 0:
     eth1Monitor = Eth1Monitor.init(
       cfg,
-      metadata.depositContractDeployedAt,
+      metadata.depositContractBlock,
+      metadata.depositContractBlockHash,
       db,
       getBeaconTime,
       config.web3Urls,

beacon_chain/nimbus_light_client.nim

@@ -91,7 +91,8 @@ programMain:
       if config.web3Urls.len > 0:
         let res = Eth1Monitor.init(
           cfg,
-          metadata.depositContractDeployedAt,
+          metadata.depositContractBlock,
+          metadata.depositContractBlockHash,
           db = nil,
           getBeaconTime,
           config.web3Urls,

beacon_chain/spec/digest.nim

@@ -154,6 +154,13 @@ proc readValue*(r: var JsonReader, a: var Eth2Digest) {.raises: [Defect, IOError
   except ValueError:
     raiseUnexpectedValue(r, "Hex string expected")
 
+func strictParse*(T: type Eth2Digest, hexStr: openArray[char]): T
+                 {.raises: [Defect, ValueError].} =
+  ## TODO We use this local definition because the string parsing functions
+  ##      provided by nimcrypto are currently too lax in their requirements
+  ##      for the input string. Invalid strings are silently ignored.
+  hexToByteArrayStrict(hexStr, result.data)
+
 func toGaugeValue*(hash: Eth2Digest): int64 =
   # Only the last 8 bytes are taken into consideration in accordance
   # to the ETH2 metrics spec:

beacon_chain/spec/presets.nim

@@ -202,7 +202,7 @@ when const_preset == "mainnet":
     # Ethereum PoW Mainnet
     DEPOSIT_CHAIN_ID: 1,
     DEPOSIT_NETWORK_ID: 1,
-    DEPOSIT_CONTRACT_ADDRESS: Eth1Address.fromHex("0x00000000219ab540356cBB839Cbe05303d7705Fa")
+    DEPOSIT_CONTRACT_ADDRESS: default(Eth1Address)
   )
 
 elif const_preset == "minimal":
@@ -307,7 +307,7 @@ elif const_preset == "minimal":
     DEPOSIT_CHAIN_ID: 5,
     DEPOSIT_NETWORK_ID: 5,
     # Configured on a per testnet basis
-    DEPOSIT_CONTRACT_ADDRESS: Eth1Address.fromHex("0x1234567890123456789012345678901234567890")
+    DEPOSIT_CONTRACT_ADDRESS: default(Eth1Address)
   )
 
 else:

research/block_sim.nim

@@ -326,7 +326,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
   var
     validatorMonitor = newClone(ValidatorMonitor.init())
     dag = ChainDAGRef.init(cfg, db, validatorMonitor, {})
-    eth1Chain = Eth1Chain.init(cfg, db)
+    eth1Chain = Eth1Chain.init(cfg, db, 0, default Eth2Digest)
     merkleizer = DepositsMerkleizer.init(depositTreeSnapshot.depositContractState)
     taskpool = Taskpool.new()
     verifier = BatchVerifier(rng: keys.newRng(), taskpool: taskpool)

vendor/eth2-networks

@@ -1 +1 @@
-Subproject commit 019c7d91e3698fd66a1950fe1067cd49dd411fe7
+Subproject commit a072c89b40cf4f28a26d047bb3aeb08995955f07

vendor/merge-testnets

@@ -1 +1 @@
-Subproject commit 8661ed6fb2600341fd8a41785b548cb08a3926c2
+Subproject commit 754aef8ab4a79f745223857e2543ab0732303fdb

vendor/nim-stew

@@ -1 +1 @@
-Subproject commit 7184d2424dc3945657884646a72715d494917aad
+Subproject commit f5846de7b2bacecd7dbff9e89d81b4f34385eb31