From 0afb5be6b435595e068e6296e8bdae2c380142fe Mon Sep 17 00:00:00 2001
From: Zahary Karadjov <zahary@gmail.com>
Date: Fri, 28 May 2021 15:42:39 +0300
Subject: [PATCH] A simple script for fetching detailed deposit data

---
 beacon_chain/eth1/eth1_monitor.nim |  14 +--
 ncli/.gitignore                    |   1 +
 ncli/deposit_downloader.nim        | 137 +++++++++++++++++++++++++++++
 ncli/download_mainnet_deposits.sh  |  13 +++
 4 files changed, 159 insertions(+), 6 deletions(-)
 create mode 100644 ncli/deposit_downloader.nim
 create mode 100755 ncli/download_mainnet_deposits.sh

diff --git a/beacon_chain/eth1/eth1_monitor.nim b/beacon_chain/eth1/eth1_monitor.nim
index c19be9e5e..b22a33c1c 100644
--- a/beacon_chain/eth1/eth1_monitor.nim
+++ b/beacon_chain/eth1/eth1_monitor.nim
@@ -114,9 +114,9 @@ type
 
   Web3DataProviderRef* = ref Web3DataProvider
 
-  DataProviderFailure = object of CatchableError
-  CorruptDataProvider = object of DataProviderFailure
-  DataProviderTimeout = object of DataProviderFailure
+  DataProviderFailure* = object of CatchableError
+  CorruptDataProvider* = object of DataProviderFailure
+  DataProviderTimeout* = object of DataProviderFailure
 
   DisconnectHandler* = proc () {.gcsafe, raises: [Defect].}
 
@@ -347,9 +347,9 @@ func hash*(x: Eth1Data): Hash =
 template hash*(x: Eth1Block): Hash =
   hash(x.voteData)
 
-template awaitWithRetries[T](lazyFutExpr: Future[T],
-                             retries = 3,
-                             timeout = web3Timeouts): untyped =
+template awaitWithRetries*[T](lazyFutExpr: Future[T],
+                              retries = 3,
+                              timeout = web3Timeouts): untyped =
   const
     reqType = astToStr(lazyFutExpr)
 
@@ -862,6 +862,8 @@ const
 proc earliestBlockOfInterest(m: Eth1Monitor): Eth1BlockNumber =
   m.latestEth1BlockNumber - (2 * m.preset.ETH1_FOLLOW_DISTANCE) - votedBlocksSafetyMargin
 
+
+
 proc syncBlockRange(m: Eth1Monitor,
                     merkleizer: ref DepositsMerkleizer,
                     fromBlock, toBlock,
diff --git a/ncli/.gitignore b/ncli/.gitignore
index f6985d352..e8f2dab9b 100644
--- a/ncli/.gitignore
+++ b/ncli/.gitignore
@@ -1,3 +1,4 @@
 ncli_pretty
 ncli_hash_tree_root
 ncli_transition
+deposit_downloader
diff --git a/ncli/deposit_downloader.nim b/ncli/deposit_downloader.nim
new file mode 100644
index 000000000..c61fadb0f
--- /dev/null
+++ b/ncli/deposit_downloader.nim
@@ -0,0 +1,137 @@
+import
+  json, strutils,
+  chronos, confutils, chronicles,
+  web3, web3/ethtypes as web3Types,
+  eth/async_utils,
+  ../beacon_chain/networking/network_metadata,
+  ../beacon_chain/eth1/eth1_monitor,
+  ../beacon_chain/spec/helpers
+
+type
+  CliFlags = object
+    web3Url {.
+      name: "web3-url".}: string
+    depositContractAddress {.
+      name: "deposit-contract".}: string
+    startBlock {.
+      name: "start-block".}: uint64
+    endBlock {.
+      name: "start-block".}: Option[uint64]
+    outDepositsFile {.
+      defaultValue: "deposits.csv"
+      name: "out-deposits-file".}: OutFile
+
+contract(DepositContract):
+  proc deposit(pubkey: Bytes48,
+               withdrawalCredentials: Bytes32,
+               signature: Bytes96,
+               deposit_data_root: FixedBytes[32])
+
+  proc get_deposit_root(): FixedBytes[32]
+  proc get_deposit_count(): Bytes8
+
+  proc DepositEvent(pubkey: Bytes48,
+                    withdrawalCredentials: Bytes32,
+                    amount: Bytes8,
+                    signature: Bytes96,
+                    index: Bytes8) {.event.}
+
+const
+  web3Timeouts = 60.seconds
+
+proc main(flags: CliFlags) {.async.} =
+  let web3 = waitFor newWeb3(flags.web3Url)
+
+  let endBlock = if flags.endBlock.isSome:
+    flags.endBlock.get
+  else:
+    awaitWithRetries(web3.provider.eth_getBlockByNumber(blockId"latest", false)).number.uint64
+
+  let depositContract = web3.contractSender(
+    DepositContract,
+    Eth1Address.fromHex flags.depositContractAddress)
+
+  var depositsFile = open(string flags.outDepositsFile, fmWrite)
+  depositsFile.write(
+    "block", ",",
+    "transaction", ",",
+    "depositor", ",",
+    "amount", ",",
+    "validatorKey", ",",
+    "withdrawalCredentials", "\n")
+
+  var currentBlock = flags.startBlock
+  while currentBlock < endBlock:
+    var
+      blocksPerRequest = 5000'u64 # This is roughly a day of Eth1 blocks
+      backoff = 100
+
+    while true:
+      let maxBlockNumberRequested = min(endBlock, currentBlock + blocksPerRequest - 1)
+
+      template retryOrRaise(err: ref CatchableError) =
+        blocksPerRequest = blocksPerRequest div 2
+        if blocksPerRequest == 0:
+          raise err
+        continue
+
+      debug "Obtaining deposit log events",
+            fromBlock = currentBlock,
+            toBlock = maxBlockNumberRequested,
+            backoff
+
+      # Reduce all request rate until we have a more general solution
+      # for dealing with Infura's rate limits
+      await sleepAsync(milliseconds(backoff))
+
+      let jsonLogsFut = depositContract.getJsonLogs(
+        DepositEvent,
+        fromBlock = some blockId(currentBlock),
+        toBlock = some blockId(maxBlockNumberRequested))
+
+      let depositLogs = try:
+        # Downloading large amounts of deposits can be quite slow
+        awaitWithTimeout(jsonLogsFut, web3Timeouts):
+          retryOrRaise newException(DataProviderTimeout,
+            "Request time out while obtaining json logs")
+      except CatchableError as err:
+        debug "Request for deposit logs failed", err = err.msg
+        backoff = (backoff * 3) div 2
+        retryOrRaise err
+
+      currentBlock = maxBlockNumberRequested + 1
+      for deposit in depositLogs:
+        let txNode = deposit{"transactionHash"}
+        if txNode != nil and txNode.kind == JString:
+          var
+            pubkey: Bytes48
+            withdrawalCredentials: Bytes32
+            amount: Bytes8
+            signature: Bytes96
+            index: Bytes8
+
+          let blockNum = parseHexInt deposit["blockNumber"].str
+          let depositData = strip0xPrefix(deposit["data"].getStr)
+          var offset = 0
+          offset += decode(depositData, offset, pubkey)
+          offset += decode(depositData, offset, withdrawalCredentials)
+          offset += decode(depositData, offset, amount)
+          offset += decode(depositData, offset, signature)
+          offset += decode(depositData, offset, index)
+
+          let txHash = TxHash.fromHex txNode.str
+          let tx = awaitWithRetries web3.provider.eth_getTransactionByHash(txHash)
+
+          depositsFile.write(
+            $blockNum, ",",
+            $txHash, ",",
+            $tx.source, ",",
+            $bytes_to_uint64(array[8, byte](amount)), ",",
+            $pubkey, ",",
+            $withdrawalCredentials, "\n")
+          depositsFile.flushFile()
+
+  info "Done"
+
+waitFor main(load CliFlags)
+
diff --git a/ncli/download_mainnet_deposits.sh b/ncli/download_mainnet_deposits.sh
new file mode 100755
index 000000000..c658645b6
--- /dev/null
+++ b/ncli/download_mainnet_deposits.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -eu
+
+cd "$(dirname "$0")"
+
+WEB3_URL=wss://mainnet.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a
+
+../env.sh nim c -r deposit_downloader.nim \
+  --web3-url="$WEB3_URL" \
+  --deposit-contract=0x00000000219ab540356cBB839Cbe05303d7705Fa \
+  --start-block=11052984
+