A simple script for fetching detailed deposit data

beacon_chain/eth1/eth1_monitor.nim

@@ -114,9 +114,9 @@ type
 
   Web3DataProviderRef* = ref Web3DataProvider
 
-  DataProviderFailure = object of CatchableError
-  CorruptDataProvider = object of DataProviderFailure
-  DataProviderTimeout = object of DataProviderFailure
+  DataProviderFailure* = object of CatchableError
+  CorruptDataProvider* = object of DataProviderFailure
+  DataProviderTimeout* = object of DataProviderFailure
 
   DisconnectHandler* = proc () {.gcsafe, raises: [Defect].}
 
@@ -347,9 +347,9 @@ func hash*(x: Eth1Data): Hash =
 template hash*(x: Eth1Block): Hash =
   hash(x.voteData)
 
-template awaitWithRetries[T](lazyFutExpr: Future[T],
-                             retries = 3,
-                             timeout = web3Timeouts): untyped =
+template awaitWithRetries*[T](lazyFutExpr: Future[T],
+                              retries = 3,
+                              timeout = web3Timeouts): untyped =
   const
     reqType = astToStr(lazyFutExpr)
 
@@ -862,6 +862,8 @@ const
 proc earliestBlockOfInterest(m: Eth1Monitor): Eth1BlockNumber =
   m.latestEth1BlockNumber - (2 * m.preset.ETH1_FOLLOW_DISTANCE) - votedBlocksSafetyMargin
 
+
+
 proc syncBlockRange(m: Eth1Monitor,
                     merkleizer: ref DepositsMerkleizer,
                     fromBlock, toBlock,

ncli/.gitignore

@@ -1,3 +1,4 @@
 ncli_pretty
 ncli_hash_tree_root
 ncli_transition
+deposit_downloader

ncli/deposit_downloader.nim

@@ -0,0 +1,137 @@
+import
+  json, strutils,
+  chronos, confutils, chronicles,
+  web3, web3/ethtypes as web3Types,
+  eth/async_utils,
+  ../beacon_chain/networking/network_metadata,
+  ../beacon_chain/eth1/eth1_monitor,
+  ../beacon_chain/spec/helpers
+
+type
+  CliFlags = object
+    web3Url {.
+      name: "web3-url".}: string
+    depositContractAddress {.
+      name: "deposit-contract".}: string
+    startBlock {.
+      name: "start-block".}: uint64
+    endBlock {.
+      name: "start-block".}: Option[uint64]
+    outDepositsFile {.
+      defaultValue: "deposits.csv"
+      name: "out-deposits-file".}: OutFile
+
+contract(DepositContract):
+  proc deposit(pubkey: Bytes48,
+               withdrawalCredentials: Bytes32,
+               signature: Bytes96,
+               deposit_data_root: FixedBytes[32])
+
+  proc get_deposit_root(): FixedBytes[32]
+  proc get_deposit_count(): Bytes8
+
+  proc DepositEvent(pubkey: Bytes48,
+                    withdrawalCredentials: Bytes32,
+                    amount: Bytes8,
+                    signature: Bytes96,
+                    index: Bytes8) {.event.}
+
+const
+  web3Timeouts = 60.seconds
+
+proc main(flags: CliFlags) {.async.} =
+  let web3 = waitFor newWeb3(flags.web3Url)
+
+  let endBlock = if flags.endBlock.isSome:
+    flags.endBlock.get
+  else:
+    awaitWithRetries(web3.provider.eth_getBlockByNumber(blockId"latest", false)).number.uint64
+
+  let depositContract = web3.contractSender(
+    DepositContract,
+    Eth1Address.fromHex flags.depositContractAddress)
+
+  var depositsFile = open(string flags.outDepositsFile, fmWrite)
+  depositsFile.write(
+    "block", ",",
+    "transaction", ",",
+    "depositor", ",",
+    "amount", ",",
+    "validatorKey", ",",
+    "withdrawalCredentials", "\n")
+
+  var currentBlock = flags.startBlock
+  while currentBlock < endBlock:
+    var
+      blocksPerRequest = 5000'u64 # This is roughly a day of Eth1 blocks
+      backoff = 100
+
+    while true:
+      let maxBlockNumberRequested = min(endBlock, currentBlock + blocksPerRequest - 1)
+
+      template retryOrRaise(err: ref CatchableError) =
+        blocksPerRequest = blocksPerRequest div 2
+        if blocksPerRequest == 0:
+          raise err
+        continue
+
+      debug "Obtaining deposit log events",
+            fromBlock = currentBlock,
+            toBlock = maxBlockNumberRequested,
+            backoff
+
+      # Reduce all request rate until we have a more general solution
+      # for dealing with Infura's rate limits
+      await sleepAsync(milliseconds(backoff))
+
+      let jsonLogsFut = depositContract.getJsonLogs(
+        DepositEvent,
+        fromBlock = some blockId(currentBlock),
+        toBlock = some blockId(maxBlockNumberRequested))
+
+      let depositLogs = try:
+        # Downloading large amounts of deposits can be quite slow
+        awaitWithTimeout(jsonLogsFut, web3Timeouts):
+          retryOrRaise newException(DataProviderTimeout,
+            "Request time out while obtaining json logs")
+      except CatchableError as err:
+        debug "Request for deposit logs failed", err = err.msg
+        backoff = (backoff * 3) div 2
+        retryOrRaise err
+
+      currentBlock = maxBlockNumberRequested + 1
+      for deposit in depositLogs:
+        let txNode = deposit{"transactionHash"}
+        if txNode != nil and txNode.kind == JString:
+          var
+            pubkey: Bytes48
+            withdrawalCredentials: Bytes32
+            amount: Bytes8
+            signature: Bytes96
+            index: Bytes8
+
+          let blockNum = parseHexInt deposit["blockNumber"].str
+          let depositData = strip0xPrefix(deposit["data"].getStr)
+          var offset = 0
+          offset += decode(depositData, offset, pubkey)
+          offset += decode(depositData, offset, withdrawalCredentials)
+          offset += decode(depositData, offset, amount)
+          offset += decode(depositData, offset, signature)
+          offset += decode(depositData, offset, index)
+
+          let txHash = TxHash.fromHex txNode.str
+          let tx = awaitWithRetries web3.provider.eth_getTransactionByHash(txHash)
+
+          depositsFile.write(
+            $blockNum, ",",
+            $txHash, ",",
+            $tx.source, ",",
+            $bytes_to_uint64(array[8, byte](amount)), ",",
+            $pubkey, ",",
+            $withdrawalCredentials, "\n")
+          depositsFile.flushFile()
+
+  info "Done"
+
+waitFor main(load CliFlags)
+

ncli/download_mainnet_deposits.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -eu
+
+cd "$(dirname "$0")"
+
+WEB3_URL=wss://mainnet.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a
+
+../env.sh nim c -r deposit_downloader.nim \
+  --web3-url="$WEB3_URL" \
+  --deposit-contract=0x00000000219ab540356cBB839Cbe05303d7705Fa \
+  --start-block=11052984
+