A simple script for fetching detailed deposit data
This commit is contained in:
parent
7f52ffb8d9
commit
0afb5be6b4
|
@ -114,9 +114,9 @@ type
|
||||||
|
|
||||||
Web3DataProviderRef* = ref Web3DataProvider
|
Web3DataProviderRef* = ref Web3DataProvider
|
||||||
|
|
||||||
DataProviderFailure = object of CatchableError
|
DataProviderFailure* = object of CatchableError
|
||||||
CorruptDataProvider = object of DataProviderFailure
|
CorruptDataProvider* = object of DataProviderFailure
|
||||||
DataProviderTimeout = object of DataProviderFailure
|
DataProviderTimeout* = object of DataProviderFailure
|
||||||
|
|
||||||
DisconnectHandler* = proc () {.gcsafe, raises: [Defect].}
|
DisconnectHandler* = proc () {.gcsafe, raises: [Defect].}
|
||||||
|
|
||||||
|
@ -347,9 +347,9 @@ func hash*(x: Eth1Data): Hash =
|
||||||
template hash*(x: Eth1Block): Hash =
|
template hash*(x: Eth1Block): Hash =
|
||||||
hash(x.voteData)
|
hash(x.voteData)
|
||||||
|
|
||||||
template awaitWithRetries[T](lazyFutExpr: Future[T],
|
template awaitWithRetries*[T](lazyFutExpr: Future[T],
|
||||||
retries = 3,
|
retries = 3,
|
||||||
timeout = web3Timeouts): untyped =
|
timeout = web3Timeouts): untyped =
|
||||||
const
|
const
|
||||||
reqType = astToStr(lazyFutExpr)
|
reqType = astToStr(lazyFutExpr)
|
||||||
|
|
||||||
|
@ -862,6 +862,8 @@ const
|
||||||
proc earliestBlockOfInterest(m: Eth1Monitor): Eth1BlockNumber =
|
proc earliestBlockOfInterest(m: Eth1Monitor): Eth1BlockNumber =
|
||||||
m.latestEth1BlockNumber - (2 * m.preset.ETH1_FOLLOW_DISTANCE) - votedBlocksSafetyMargin
|
m.latestEth1BlockNumber - (2 * m.preset.ETH1_FOLLOW_DISTANCE) - votedBlocksSafetyMargin
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
proc syncBlockRange(m: Eth1Monitor,
|
proc syncBlockRange(m: Eth1Monitor,
|
||||||
merkleizer: ref DepositsMerkleizer,
|
merkleizer: ref DepositsMerkleizer,
|
||||||
fromBlock, toBlock,
|
fromBlock, toBlock,
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
ncli_pretty
|
ncli_pretty
|
||||||
ncli_hash_tree_root
|
ncli_hash_tree_root
|
||||||
ncli_transition
|
ncli_transition
|
||||||
|
deposit_downloader
|
||||||
|
|
|
@ -0,0 +1,137 @@
|
||||||
|
import
|
||||||
|
json, strutils,
|
||||||
|
chronos, confutils, chronicles,
|
||||||
|
web3, web3/ethtypes as web3Types,
|
||||||
|
eth/async_utils,
|
||||||
|
../beacon_chain/networking/network_metadata,
|
||||||
|
../beacon_chain/eth1/eth1_monitor,
|
||||||
|
../beacon_chain/spec/helpers
|
||||||
|
|
||||||
|
type
|
||||||
|
CliFlags = object
|
||||||
|
web3Url {.
|
||||||
|
name: "web3-url".}: string
|
||||||
|
depositContractAddress {.
|
||||||
|
name: "deposit-contract".}: string
|
||||||
|
startBlock {.
|
||||||
|
name: "start-block".}: uint64
|
||||||
|
endBlock {.
|
||||||
|
name: "start-block".}: Option[uint64]
|
||||||
|
outDepositsFile {.
|
||||||
|
defaultValue: "deposits.csv"
|
||||||
|
name: "out-deposits-file".}: OutFile
|
||||||
|
|
||||||
|
contract(DepositContract):
|
||||||
|
proc deposit(pubkey: Bytes48,
|
||||||
|
withdrawalCredentials: Bytes32,
|
||||||
|
signature: Bytes96,
|
||||||
|
deposit_data_root: FixedBytes[32])
|
||||||
|
|
||||||
|
proc get_deposit_root(): FixedBytes[32]
|
||||||
|
proc get_deposit_count(): Bytes8
|
||||||
|
|
||||||
|
proc DepositEvent(pubkey: Bytes48,
|
||||||
|
withdrawalCredentials: Bytes32,
|
||||||
|
amount: Bytes8,
|
||||||
|
signature: Bytes96,
|
||||||
|
index: Bytes8) {.event.}
|
||||||
|
|
||||||
|
const
|
||||||
|
web3Timeouts = 60.seconds
|
||||||
|
|
||||||
|
proc main(flags: CliFlags) {.async.} =
|
||||||
|
let web3 = waitFor newWeb3(flags.web3Url)
|
||||||
|
|
||||||
|
let endBlock = if flags.endBlock.isSome:
|
||||||
|
flags.endBlock.get
|
||||||
|
else:
|
||||||
|
awaitWithRetries(web3.provider.eth_getBlockByNumber(blockId"latest", false)).number.uint64
|
||||||
|
|
||||||
|
let depositContract = web3.contractSender(
|
||||||
|
DepositContract,
|
||||||
|
Eth1Address.fromHex flags.depositContractAddress)
|
||||||
|
|
||||||
|
var depositsFile = open(string flags.outDepositsFile, fmWrite)
|
||||||
|
depositsFile.write(
|
||||||
|
"block", ",",
|
||||||
|
"transaction", ",",
|
||||||
|
"depositor", ",",
|
||||||
|
"amount", ",",
|
||||||
|
"validatorKey", ",",
|
||||||
|
"withdrawalCredentials", "\n")
|
||||||
|
|
||||||
|
var currentBlock = flags.startBlock
|
||||||
|
while currentBlock < endBlock:
|
||||||
|
var
|
||||||
|
blocksPerRequest = 5000'u64 # This is roughly a day of Eth1 blocks
|
||||||
|
backoff = 100
|
||||||
|
|
||||||
|
while true:
|
||||||
|
let maxBlockNumberRequested = min(endBlock, currentBlock + blocksPerRequest - 1)
|
||||||
|
|
||||||
|
template retryOrRaise(err: ref CatchableError) =
|
||||||
|
blocksPerRequest = blocksPerRequest div 2
|
||||||
|
if blocksPerRequest == 0:
|
||||||
|
raise err
|
||||||
|
continue
|
||||||
|
|
||||||
|
debug "Obtaining deposit log events",
|
||||||
|
fromBlock = currentBlock,
|
||||||
|
toBlock = maxBlockNumberRequested,
|
||||||
|
backoff
|
||||||
|
|
||||||
|
# Reduce all request rate until we have a more general solution
|
||||||
|
# for dealing with Infura's rate limits
|
||||||
|
await sleepAsync(milliseconds(backoff))
|
||||||
|
|
||||||
|
let jsonLogsFut = depositContract.getJsonLogs(
|
||||||
|
DepositEvent,
|
||||||
|
fromBlock = some blockId(currentBlock),
|
||||||
|
toBlock = some blockId(maxBlockNumberRequested))
|
||||||
|
|
||||||
|
let depositLogs = try:
|
||||||
|
# Downloading large amounts of deposits can be quite slow
|
||||||
|
awaitWithTimeout(jsonLogsFut, web3Timeouts):
|
||||||
|
retryOrRaise newException(DataProviderTimeout,
|
||||||
|
"Request time out while obtaining json logs")
|
||||||
|
except CatchableError as err:
|
||||||
|
debug "Request for deposit logs failed", err = err.msg
|
||||||
|
backoff = (backoff * 3) div 2
|
||||||
|
retryOrRaise err
|
||||||
|
|
||||||
|
currentBlock = maxBlockNumberRequested + 1
|
||||||
|
for deposit in depositLogs:
|
||||||
|
let txNode = deposit{"transactionHash"}
|
||||||
|
if txNode != nil and txNode.kind == JString:
|
||||||
|
var
|
||||||
|
pubkey: Bytes48
|
||||||
|
withdrawalCredentials: Bytes32
|
||||||
|
amount: Bytes8
|
||||||
|
signature: Bytes96
|
||||||
|
index: Bytes8
|
||||||
|
|
||||||
|
let blockNum = parseHexInt deposit["blockNumber"].str
|
||||||
|
let depositData = strip0xPrefix(deposit["data"].getStr)
|
||||||
|
var offset = 0
|
||||||
|
offset += decode(depositData, offset, pubkey)
|
||||||
|
offset += decode(depositData, offset, withdrawalCredentials)
|
||||||
|
offset += decode(depositData, offset, amount)
|
||||||
|
offset += decode(depositData, offset, signature)
|
||||||
|
offset += decode(depositData, offset, index)
|
||||||
|
|
||||||
|
let txHash = TxHash.fromHex txNode.str
|
||||||
|
let tx = awaitWithRetries web3.provider.eth_getTransactionByHash(txHash)
|
||||||
|
|
||||||
|
depositsFile.write(
|
||||||
|
$blockNum, ",",
|
||||||
|
$txHash, ",",
|
||||||
|
$tx.source, ",",
|
||||||
|
$bytes_to_uint64(array[8, byte](amount)), ",",
|
||||||
|
$pubkey, ",",
|
||||||
|
$withdrawalCredentials, "\n")
|
||||||
|
depositsFile.flushFile()
|
||||||
|
|
||||||
|
info "Done"
|
||||||
|
|
||||||
|
waitFor main(load CliFlags)
|
||||||
|
|
|
@ -0,0 +1,13 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
|
WEB3_URL=wss://mainnet.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a
|
||||||
|
|
||||||
|
../env.sh nim c -r deposit_downloader.nim \
|
||||||
|
--web3-url="$WEB3_URL" \
|
||||||
|
--deposit-contract=0x00000000219ab540356cBB839Cbe05303d7705Fa \
|
||||||
|
--start-block=11052984
|
||||||
|
|
Loading…
Reference in New Issue