status-im
/
nimbus-eth2
mirror of https://github.com/status-im/nimbus-eth2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add Eth1 deposits simulation to block_sim

This commit is contained in:
Zahary Karadjov 2020-12-03 06:30:35 +02:00 committed by zah
parent 52ca12c79f
commit 338428cbd7
9 changed files with 217 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
beacon_chain/eth1_monitor.nim
View File

@ -55,8 +55,12 @@ type
activeValidatorsCount*: uint64 activeValidatorsCount*: uint64
Eth1Chain* = object Eth1Chain* = object
db: BeaconChainDB
preset: RuntimePreset
blocks: Deque[Eth1Block] blocks: Deque[Eth1Block]
blocksByHash: Table[BlockHash, Eth1Block] blocksByHash: Table[BlockHash, Eth1Block]
finalizedBlockHash: Eth2Digest
finalizedDepositsMerkleizer: DepositsMerkleizer
Eth1MonitorState = enum Eth1MonitorState = enum
Initialized Initialized
@ -67,21 +71,16 @@ type
Eth1Monitor* = ref object Eth1Monitor* = ref object
state: Eth1MonitorState state: Eth1MonitorState
preset: RuntimePreset
web3Url: string web3Url: string
eth1Network: Option[Eth1Network] eth1Network: Option[Eth1Network]
depositContractAddress*: Eth1Address depositContractAddress*: Eth1Address
dataProvider: Web3DataProviderRef dataProvider: Web3DataProviderRef
eth1Chain: Eth1Chain
latestEth1BlockNumber: Eth1BlockNumber latestEth1BlockNumber: Eth1BlockNumber
eth1Progress: AsyncEvent eth1Progress: AsyncEvent
db: BeaconChainDB
eth1Chain: Eth1Chain
knownStart: DepositContractSnapshot
eth2FinalizedDepositsMerkleizer: DepositsMerkleizer
runFut: Future[void] runFut: Future[void]
stopFut: Future[void] stopFut: Future[void]
@ -213,6 +212,15 @@ when hasGenesisDetection:
template blocks*(m: Eth1Monitor): Deque[Eth1Block] = template blocks*(m: Eth1Monitor): Deque[Eth1Block] =
m.eth1Chain.blocks m.eth1Chain.blocks
template db(m: Eth1Monitor): auto =
m.eth1Chain.db
template preset(m: Eth1Monitor): auto =
m.eth1Chain.preset
template finalizedDepositsMerkleizer(m: Eth1Monitor): auto =
m.eth1Chain.finalizedDepositsMerkleizer
proc fixupWeb3Urls*(web3Url: var string) = proc fixupWeb3Urls*(web3Url: var string) =
## Converts HTTP and HTTPS Infura URLs to their WebSocket equivalents ## Converts HTTP and HTTPS Infura URLs to their WebSocket equivalents
## because we are missing a functional HTTPS client. ## because we are missing a functional HTTPS client.
@ -291,8 +299,8 @@ template asBlockHash(x: Eth2Digest): BlockHash =
func shortLog*(b: Eth1Block): string = func shortLog*(b: Eth1Block): string =
&"{b.number}:{shortLog b.voteData.block_hash}(deposits = {b.voteData.deposit_count})" &"{b.number}:{shortLog b.voteData.block_hash}(deposits = {b.voteData.deposit_count})"
template findBlock*(eth1Chain: Eth1Chain, eth1Data: Eth1Data): Eth1Block = template findBlock*(chain: Eth1Chain, eth1Data: Eth1Data): Eth1Block =
getOrDefault(eth1Chain.blocksByHash, asBlockHash(eth1Data.block_hash), nil) getOrDefault(chain.blocksByHash, asBlockHash(eth1Data.block_hash), nil)
func makeSuccessorWithoutDeposits(existingBlock: Eth1Block, func makeSuccessorWithoutDeposits(existingBlock: Eth1Block,
successor: BlockObject): ETh1Block = successor: BlockObject): ETh1Block =
@ -307,21 +315,21 @@ func makeSuccessorWithoutDeposits(existingBlock: Eth1Block,
when hasGenesisDetection: when hasGenesisDetection:
result.activeValidatorsCount = existingBlock.activeValidatorsCount result.activeValidatorsCount = existingBlock.activeValidatorsCount
func latestCandidateBlock(m: Eth1Monitor, periodStart: uint64): Eth1Block = func latestCandidateBlock(chain: Eth1Chain, periodStart: uint64): Eth1Block =
for i in countdown(m.eth1Chain.blocks.len - 1, 0): for i in countdown(chain.blocks.len - 1, 0):
let blk = m.eth1Chain.blocks[i] let blk = chain.blocks[i]
if is_candidate_block(m.preset, blk, periodStart): if is_candidate_block(chain.preset, blk, periodStart):
return blk return blk
proc popFirst(eth1Chain: var Eth1Chain) = proc popFirst(chain: var Eth1Chain) =
let removed = eth1Chain.blocks.popFirst let removed = chain.blocks.popFirst
eth1Chain.blocksByHash.del removed.voteData.block_hash.asBlockHash chain.blocksByHash.del removed.voteData.block_hash.asBlockHash
eth1_chain_len.set eth1Chain.blocks.len.int64 eth1_chain_len.set chain.blocks.len.int64
proc addBlock(eth1Chain: var Eth1Chain, newBlock: Eth1Block) = proc addBlock*(chain: var Eth1Chain, newBlock: Eth1Block) =
eth1Chain.blocks.addLast newBlock chain.blocks.addLast newBlock
eth1Chain.blocksByHash[newBlock.voteData.block_hash.asBlockHash] = newBlock chain.blocksByHash[newBlock.voteData.block_hash.asBlockHash] = newBlock
eth1_chain_len.set eth1Chain.blocks.len.int64 eth1_chain_len.set chain.blocks.len.int64
func hash*(x: Eth1Data): Hash = func hash*(x: Eth1Data): Hash =
hashData(unsafeAddr x, sizeof(x)) hashData(unsafeAddr x, sizeof(x))
@ -495,20 +503,21 @@ proc onBlockHeaders*(p: Web3DataProviderRef,
{.push raises: [Defect].} {.push raises: [Defect].}
func getDepositsRoot(m: DepositsMerkleizer): Eth2Digest = func getDepositsRoot*(m: DepositsMerkleizer): Eth2Digest =
mixInLength(m.getFinalHash, int m.totalChunks) mixInLength(m.getFinalHash, int m.totalChunks)
func toDepositContractState(merkleizer: DepositsMerkleizer): DepositContractState = func toDepositContractState*(merkleizer: DepositsMerkleizer): DepositContractState =
# TODO There is an off by one discrepancy in the size of the arrays here that # TODO There is an off by one discrepancy in the size of the arrays here that
# need to be investigated. It shouldn't matter as long as the tree is # need to be investigated. It shouldn't matter as long as the tree is
# not populated to its maximum size. # not populated to its maximum size.
result.branch[0..31] = merkleizer.getCombinedChunks[0..31] result.branch[0..31] = merkleizer.getCombinedChunks[0..31]
result.deposit_count[24..31] = merkleizer.getChunkCount().toBytesBE result.deposit_count[24..31] = merkleizer.getChunkCount().toBytesBE
func createMerkleizer(s: DepositContractSnapshot): DepositsMerkleizer = func createMerkleizer*(s: DepositContractState): DepositsMerkleizer =
DepositsMerkleizer.init( DepositsMerkleizer.init(s.branch, s.depositCountU64)
s.depositContractState.branch,
s.depositContractState.depositCountU64) func createMerkleizer*(s: DepositContractSnapshot): DepositsMerkleizer =
createMerkleizer(s.depositContractState)
func eth1DataFromMerkleizer(eth1Block: Eth2Digest, func eth1DataFromMerkleizer(eth1Block: Eth2Digest,
merkleizer: DepositsMerkleizer): Eth1Data = merkleizer: DepositsMerkleizer): Eth1Data =
@ -517,24 +526,24 @@ func eth1DataFromMerkleizer(eth1Block: Eth2Digest,
deposit_count: merkleizer.getChunkCount, deposit_count: merkleizer.getChunkCount,
deposit_root: merkleizer.getDepositsRoot) deposit_root: merkleizer.getDepositsRoot)
proc pruneOldBlocks(m: Eth1Monitor, depositIndex: uint64) = proc pruneOldBlocks(chain: var Eth1Chain, depositIndex: uint64) =
let initialChunks = m.eth2FinalizedDepositsMerkleizer.getChunkCount let initialChunks = chain.finalizedDepositsMerkleizer.getChunkCount
var lastBlock: Eth1Block var lastBlock: Eth1Block
while m.eth1Chain.blocks.len > 0: while chain.blocks.len > 0:
let blk = m.eth1Chain.blocks.peekFirst let blk = chain.blocks.peekFirst
if blk.voteData.deposit_count >= depositIndex: if blk.voteData.deposit_count >= depositIndex:
break break
else: else:
for deposit in blk.deposits: for deposit in blk.deposits:
m.eth2FinalizedDepositsMerkleizer.addChunk hash_tree_root(deposit).data chain.finalizedDepositsMerkleizer.addChunk hash_tree_root(deposit).data
m.eth1Chain.popFirst() chain.popFirst()
lastBlock = blk lastBlock = blk
if m.eth2FinalizedDepositsMerkleizer.getChunkCount > initialChunks: if chain.finalizedDepositsMerkleizer.getChunkCount > initialChunks:
m.db.putEth2FinalizedTo DepositContractSnapshot( chain.db.putEth2FinalizedTo DepositContractSnapshot(
eth1Block: lastBlock.voteData.block_hash, eth1Block: lastBlock.voteData.block_hash,
depositContractState: m.eth2FinalizedDepositsMerkleizer.toDepositContractState) depositContractState: chain.finalizedDepositsMerkleizer.toDepositContractState)
eth1_finalized_head.set lastBlock.number.toGaugeValue eth1_finalized_head.set lastBlock.number.toGaugeValue
eth1_finalized_deposits.set lastBlock.voteData.deposit_count.toGaugeValue eth1_finalized_deposits.set lastBlock.voteData.deposit_count.toGaugeValue
@ -543,24 +552,24 @@ proc pruneOldBlocks(m: Eth1Monitor, depositIndex: uint64) =
newTailBlock = lastBlock.voteData.block_hash, newTailBlock = lastBlock.voteData.block_hash,
depositsCount = lastBlock.voteData.deposit_count depositsCount = lastBlock.voteData.deposit_count
proc advanceMerkleizer(eth1Chain: Eth1Chain, proc advanceMerkleizer(chain: Eth1Chain,
merkleizer: var DepositsMerkleizer, merkleizer: var DepositsMerkleizer,
depositIndex: uint64): bool = depositIndex: uint64): bool =
if eth1Chain.blocks.len == 0: if chain.blocks.len == 0:
return depositIndex == merkleizer.getChunkCount return depositIndex == merkleizer.getChunkCount
if eth1Chain.blocks.peekLast.voteData.deposit_count < depositIndex: if chain.blocks.peekLast.voteData.deposit_count < depositIndex:
return false return false
let let
firstBlock = eth1Chain.blocks[0] firstBlock = chain.blocks[0]
depositsInLastPrunedBlock = firstBlock.voteData.deposit_count - depositsInLastPrunedBlock = firstBlock.voteData.deposit_count -
firstBlock.deposits.lenu64 firstBlock.deposits.lenu64
# advanceMerkleizer should always be called shortly after prunning the chain # advanceMerkleizer should always be called shortly after prunning the chain
doAssert depositsInLastPrunedBlock == merkleizer.getChunkCount doAssert depositsInLastPrunedBlock == merkleizer.getChunkCount
for blk in eth1Chain.blocks: for blk in chain.blocks:
for deposit in blk.deposits: for deposit in blk.deposits:
if merkleizer.getChunkCount < depositIndex: if merkleizer.getChunkCount < depositIndex:
merkleizer.addChunk hash_tree_root(deposit).data merkleizer.addChunk hash_tree_root(deposit).data
@ -569,13 +578,13 @@ proc advanceMerkleizer(eth1Chain: Eth1Chain,
return merkleizer.getChunkCount == depositIndex return merkleizer.getChunkCount == depositIndex
proc getDepositsRange(eth1Chain: Eth1Chain, first, last: uint64): seq[DepositData] = proc getDepositsRange(chain: Eth1Chain, first, last: uint64): seq[DepositData] =
# TODO It's possible to make this faster by performing binary search that # TODO It's possible to make this faster by performing binary search that
# will locate the blocks holding the `first` and `last` indices. # will locate the blocks holding the `first` and `last` indices.
# TODO There is an assumption here that the requested range will be present # TODO There is an assumption here that the requested range will be present
# in the Eth1Chain. This should hold true at the single call site right # in the Eth1Chain. This should hold true at the single call site right
# now, but we need to guard the pre-conditions better. # now, but we need to guard the pre-conditions better.
for blk in eth1Chain.blocks: for blk in chain.blocks:
if blk.voteData.deposit_count <= first: if blk.voteData.deposit_count <= first:
continue continue
@ -597,64 +606,72 @@ proc lowerBound(chain: Eth1Chain, depositCount: uint64): Eth1Block =
return return
result = eth1Block result = eth1Block
proc trackFinalizedState*(m: Eth1Monitor, proc trackFinalizedState*(chain: var Eth1Chain,
finalizedEth1Data: Eth1Data, finalizedEth1Data: Eth1Data,
finalizedStateDepositIndex: uint64): bool = finalizedStateDepositIndex: uint64): bool =
# Returns true if the Eth1Monitor is synced to the finalization point # Returns true if the Eth1Monitor is synced to the finalization point
if m.eth1Chain.blocks.len == 0: if chain.blocks.len == 0:
debug "Eth1 chain not initialized" debug "Eth1 chain not initialized"
return false return false
let latest = m.eth1Chain.blocks.peekLast let latest = chain.blocks.peekLast
if latest.voteData.deposit_count < finalizedEth1Data.deposit_count: if latest.voteData.deposit_count < finalizedEth1Data.deposit_count:
debug "Eth1 chain not synced", debug "Eth1 chain not synced",
ourDepositsCount = latest.voteData.deposit_count, ourDepositsCount = latest.voteData.deposit_count,
targetDepositsCount = finalizedEth1Data.deposit_count targetDepositsCount = finalizedEth1Data.deposit_count
return false return false
let matchingBlock = m.eth1Chain.lowerBound(finalizedEth1Data.deposit_count) let matchingBlock = chain.lowerBound(finalizedEth1Data.deposit_count)
result = if matchingBlock != nil: result = if matchingBlock != nil:
if matchingBlock.voteData.deposit_root == finalizedEth1Data.deposit_root: if matchingBlock.voteData.deposit_root == finalizedEth1Data.deposit_root:
matchingBlock.voteDataVerified = true matchingBlock.voteDataVerified = true
true true
else: else:
error "Corrupted deposits history detected", error "Corrupted deposits history detected",
depositsCount = finalizedEth1Data.deposit_count, ourDepositsCount = matchingBlock.voteData.deposit_count,
targetDepositsRoot = finalizedEth1Data.deposit_root, taretDepositsCount = finalizedEth1Data.deposit_count,
ourDepositsRoot = matchingBlock.voteData.deposit_root ourDepositsRoot = matchingBlock.voteData.deposit_root,
targetDepositsRoot = finalizedEth1Data.deposit_root
false false
else: else:
error "The Eth1 chain is in inconsistent state", error "The Eth1 chain is in inconsistent state",
checkpointHash = finalizedEth1Data.block_hash, checkpointHash = finalizedEth1Data.block_hash,
checkpointDeposits = finalizedEth1Data.deposit_count, checkpointDeposits = finalizedEth1Data.deposit_count,
localChainStart = shortLog(m.eth1Chain.blocks.peekFirst), localChainStart = shortLog(chain.blocks.peekFirst),
localChainEnd = shortLog(m.eth1Chain.blocks.peekLast) localChainEnd = shortLog(chain.blocks.peekLast)
false false
if result: if result:
m.pruneOldBlocks(finalizedStateDepositIndex) chain.pruneOldBlocks(finalizedStateDepositIndex)
template trackFinalizedState*(m: Eth1Monitor,
finalizedEth1Data: Eth1Data,
finalizedStateDepositIndex: uint64): bool =
trackFinalizedState(m.eth1Chain, finalizedEth1Data, finalizedStateDepositIndex)
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/validator.md#get_eth1_data # https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/validator.md#get_eth1_data
proc getBlockProposalData*(m: Eth1Monitor, proc getBlockProposalData*(chain: var Eth1Chain,
state: BeaconState, state: BeaconState,
finalizedEth1Data: Eth1Data, finalizedEth1Data: Eth1Data,
finalizedStateDepositIndex: uint64): BlockProposalEth1Data = finalizedStateDepositIndex: uint64): BlockProposalEth1Data =
let let
periodStart = voting_period_start_time(state) periodStart = voting_period_start_time(state)
hasLatestDeposits = m.trackFinalizedState(finalizedEth1Data, hasLatestDeposits = chain.trackFinalizedState(finalizedEth1Data,
finalizedStateDepositIndex) finalizedStateDepositIndex)
var otherVotesCountTable = initCountTable[Eth1Data]() var otherVotesCountTable = initCountTable[Eth1Data]()
for vote in state.eth1_data_votes: for vote in state.eth1_data_votes:
let eth1Block = chain.findBlock(vote)
if eth1Block == nil:
continue
let let
eth1Block = m.eth1Chain.findBlock(vote)
isSuccessor = vote.deposit_count >= state.eth1_data.deposit_count isSuccessor = vote.deposit_count >= state.eth1_data.deposit_count
# TODO(zah) # TODO(zah)
# There is a slight deviation from the spec here to deal with the following # There is a slight deviation from the spec here to deal with the following
# problem: the in-memory database of eth1 blocks for a restarted node will # problem: the in-memory database of eth1 blocks for a restarted node will
# be empty which will lead a "no change" vote. To fix this, we'll need to # be empty which will lead a "no change" vote. To fix this, we'll need to
# add rolling persistance for all potentially voted on blocks. # add rolling persistance for all potentially voted on blocks.
isCandidate = (eth1Block == nil or is_candidate_block(m.preset, eth1Block, periodStart)) isCandidate = (is_candidate_block(chain.preset, eth1Block, periodStart))
if isSuccessor and isCandidate: if isSuccessor and isCandidate:
otherVotesCountTable.inc vote otherVotesCountTable.inc vote
@ -669,7 +686,7 @@ proc getBlockProposalData*(m: Eth1Monitor,
if uint64((votes + 1) * 2) > SLOTS_PER_ETH1_VOTING_PERIOD: if uint64((votes + 1) * 2) > SLOTS_PER_ETH1_VOTING_PERIOD:
pendingDepositsCount = winningVote.deposit_count - state.eth1_deposit_index pendingDepositsCount = winningVote.deposit_count - state.eth1_deposit_index
else: else:
let latestBlock = m.latestCandidateBlock(periodStart) let latestBlock = chain.latestCandidateBlock(periodStart)
if latestBlock == nil: if latestBlock == nil:
debug "No acceptable eth1 votes and no recent candidates. Voting no change" debug "No acceptable eth1 votes and no recent candidates. Voting no change"
result.vote = state.eth1_data result.vote = state.eth1_data
@ -681,13 +698,13 @@ proc getBlockProposalData*(m: Eth1Monitor,
if hasLatestDeposits: if hasLatestDeposits:
let let
totalDepositsInNewBlock = min(MAX_DEPOSITS, pendingDepositsCount) totalDepositsInNewBlock = min(MAX_DEPOSITS, pendingDepositsCount)
deposits = m.eth1Chain.getDepositsRange( deposits = chain.getDepositsRange(
state.eth1_deposit_index, state.eth1_deposit_index,
state.eth1_deposit_index + pendingDepositsCount) state.eth1_deposit_index + pendingDepositsCount)
depositRoots = mapIt(deposits, hash_tree_root(it)) depositRoots = mapIt(deposits, hash_tree_root(it))
var scratchMerkleizer = copy m.eth2FinalizedDepositsMerkleizer var scratchMerkleizer = copy chain.finalizedDepositsMerkleizer
if m.eth1Chain.advanceMerkleizer(scratchMerkleizer, state.eth1_deposit_index): if chain.advanceMerkleizer(scratchMerkleizer, state.eth1_deposit_index):
let proofs = scratchMerkleizer.addChunksAndGenMerkleProofs(depositRoots) let proofs = scratchMerkleizer.addChunksAndGenMerkleProofs(depositRoots)
for i in 0 ..< totalDepositsInNewBlock: for i in 0 ..< totalDepositsInNewBlock:
var proof: array[33, Eth2Digest] var proof: array[33, Eth2Digest]
@ -701,6 +718,12 @@ proc getBlockProposalData*(m: Eth1Monitor,
else: else:
result.hasMissingDeposits = true result.hasMissingDeposits = true
template getBlockProposalData*(m: Eth1Monitor,
state: BeaconState,
finalizedEth1Data: Eth1Data,
finalizedStateDepositIndex: uint64): BlockProposalEth1Data =
getBlockProposalData(m.eth1Chain, state, finalizedEth1Data, finalizedStateDepositIndex)
{.pop.} {.pop.}
proc new(T: type Web3DataProvider, proc new(T: type Web3DataProvider,
@ -718,9 +741,28 @@ proc new(T: type Web3DataProvider,
return ok Web3DataProviderRef(url: web3Url, web3: web3, ns: ns) return ok Web3DataProviderRef(url: web3Url, web3: web3, ns: ns)
proc putInitialDepositContractSnapshot*(db: BeaconChainDB,
s: DepositContractSnapshot) =
let existingStart = db.getEth2FinalizedTo()
if not existingStart.isOk:
db.putEth2FinalizedTo(s)
template getOrDefault[T, E](r: Result[T, E]): T =
type TT = T
get(r, default(TT))
proc init*(T: type Eth1Chain, preset: RuntimePreset, db: BeaconChainDB): T =
let finalizedDeposits = db.getEth2FinalizedTo().getOrDefault()
let m = finalizedDeposits.createMerkleizer
T(db: db,
preset: preset,
finalizedBlockHash: finalizedDeposits.eth1Block,
finalizedDepositsMerkleizer: finalizedDeposits.createMerkleizer)
proc init*(T: type Eth1Monitor, proc init*(T: type Eth1Monitor,
db: BeaconChainDB,
preset: RuntimePreset, preset: RuntimePreset,
db: BeaconChainDB,
web3Url: string, web3Url: string,
depositContractAddress: Eth1Address, depositContractAddress: Eth1Address,
depositContractSnapshot: DepositContractSnapshot, depositContractSnapshot: DepositContractSnapshot,
@ -728,10 +770,10 @@ proc init*(T: type Eth1Monitor,
var web3Url = web3Url var web3Url = web3Url
fixupWeb3Urls web3Url fixupWeb3Urls web3Url
putInitialDepositContractSnapshot(db, depositContractSnapshot)
T(state: Initialized, T(state: Initialized,
db: db, eth1Chain: Eth1Chain.init(preset, db),
preset: preset,
knownStart: depositContractSnapshot,
depositContractAddress: depositContractAddress, depositContractAddress: depositContractAddress,
web3Url: web3Url, web3Url: web3Url,
eth1Network: eth1Network, eth1Network: eth1Network,
@ -985,30 +1027,24 @@ proc startEth1Syncing(m: Eth1Monitor, delayBeforeStart: Duration) {.async.} =
do (err: CatchableError): do (err: CatchableError):
debug "Error while processing Eth1 block headers subscription", err = err.msg debug "Error while processing Eth1 block headers subscription", err = err.msg
let eth2PreviouslyFinalizedTo = m.db.getEth2FinalizedTo()
if eth2PreviouslyFinalizedTo.isOk:
m.knownStart = eth2PreviouslyFinalizedTo.get
m.eth2FinalizedDepositsMerkleizer = m.knownStart.createMerkleizer
let startBlock = awaitWithRetries( let startBlock = awaitWithRetries(
m.dataProvider.getBlockByHash(m.knownStart.eth1Block.asBlockHash)) m.dataProvider.getBlockByHash(m.eth1Chain.finalizedBlockHash.asBlockHash))
doAssert m.eth1Chain.blocks.len == 0 doAssert m.eth1Chain.blocks.len == 0
m.eth1Chain.addBlock Eth1Block( m.eth1Chain.addBlock Eth1Block(
number: Eth1BlockNumber startBlock.number, number: Eth1BlockNumber startBlock.number,
timestamp: Eth1BlockTimestamp startBlock.timestamp, timestamp: Eth1BlockTimestamp startBlock.timestamp,
voteData: eth1DataFromMerkleizer( voteData: eth1DataFromMerkleizer(
m.knownStart.eth1Block, m.eth1Chain.finalizedBlockHash,
m.eth2FinalizedDepositsMerkleizer)) m.eth1Chain.finalizedDepositsMerkleizer))
var eth1SyncedTo = Eth1BlockNumber startBlock.number var eth1SyncedTo = Eth1BlockNumber startBlock.number
eth1_synced_head.set eth1SyncedTo.toGaugeValue eth1_synced_head.set eth1SyncedTo.toGaugeValue
eth1_finalized_head.set eth1SyncedTo.toGaugeValue eth1_finalized_head.set eth1SyncedTo.toGaugeValue
eth1_finalized_deposits.set( eth1_finalized_deposits.set(
m.eth2FinalizedDepositsMerkleizer.getChunkCount.toGaugeValue) m.eth1Chain.finalizedDepositsMerkleizer.getChunkCount.toGaugeValue)
var scratchMerkleizer = newClone(copy m.eth2FinalizedDepositsMerkleizer) var scratchMerkleizer = newClone(copy m.finalizedDepositsMerkleizer)
debug "Starting Eth1 syncing", `from` = shortLog(m.eth1Chain.blocks[0]) debug "Starting Eth1 syncing", `from` = shortLog(m.eth1Chain.blocks[0])

4
beacon_chain/nimbus_beacon_node.nim
View File

@ -140,8 +140,8 @@ proc init*(T: type BeaconNode,
# TODO Could move this to a separate "GenesisMonitor" process or task # TODO Could move this to a separate "GenesisMonitor" process or task
# that would do only this - see Paul's proposal for this. # that would do only this - see Paul's proposal for this.
let eth1MonitorRes = await Eth1Monitor.init( let eth1MonitorRes = await Eth1Monitor.init(
db,
conf.runtimePreset, conf.runtimePreset,
db,
conf.web3Url, conf.web3Url,
depositContractAddress, depositContractAddress,
depositContractDeployedAt, depositContractDeployedAt,
@ -254,8 +254,8 @@ proc init*(T: type BeaconNode,
let genesisDepositsSnapshot = SSZ.decode(genesisDepositsSnapshotContents[], let genesisDepositsSnapshot = SSZ.decode(genesisDepositsSnapshotContents[],
DepositContractSnapshot) DepositContractSnapshot)
eth1Monitor = Eth1Monitor.init( eth1Monitor = Eth1Monitor.init(
db,
conf.runtimePreset, conf.runtimePreset,
db,
conf.web3Url, conf.web3Url,
depositContractAddress, depositContractAddress,
genesisDepositsSnapshot, genesisDepositsSnapshot,

1
beacon_chain/rpc/nimbus_api.nim
View File

@ -23,7 +23,6 @@ logScope: topics = "nimbusapi"
type type
RpcServer = RpcHttpServer RpcServer = RpcHttpServer
Eth1Block = eth1_monitor.Eth1Block
FutureInfo* = object FutureInfo* = object
id*: int id*: int

7
beacon_chain/spec/datatypes.nim
View File

@ -404,13 +404,6 @@ type
deposit_count*: uint64 deposit_count*: uint64
block_hash*: Eth2Digest block_hash*: Eth2Digest
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/validator.md#eth1block
Eth1Block* = object
timestamp*: uint64
deposit_root*: Eth2Digest
deposit_count*: uint64
# All other eth1 block fields
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#signedvoluntaryexit # https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#signedvoluntaryexit
SignedVoluntaryExit* = object SignedVoluntaryExit* = object
message*: VoluntaryExit message*: VoluntaryExit

74
research/block_sim.nim
View File

@ -15,15 +15,15 @@
# a database, as if a real node was running. # a database, as if a real node was running.
import import
confutils, chronicles, stats, times, strformat, math, stats, times, strformat,
options, random, tables, os, options, random, tables, os,
eth/db/kvstore_sqlite3, confutils, chronicles, eth/db/kvstore_sqlite3,
../tests/[testblockutil], ../tests/[testblockutil],
../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, presets, ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, presets,
helpers, validator, signatures, state_transition], helpers, validator, signatures, state_transition],
../beacon_chain/[ ../beacon_chain/[
attestation_pool, beacon_node_types, beacon_chain_db, attestation_pool, beacon_node_types, beacon_chain_db,
interop, validator_pool], validator_pool, eth1_monitor, extras],
../beacon_chain/block_pools/[ ../beacon_chain/block_pools/[
spec_cache, chain_dag, quarantine, clearance], spec_cache, chain_dag, quarantine, clearance],
../beacon_chain/ssz/[merkleization, ssz_serialization], ../beacon_chain/ssz/[merkleization, ssz_serialization],
@ -37,6 +37,18 @@ type Timers = enum
tAttest = "Have committee attest to block" tAttest = "Have committee attest to block"
tReplay = "Replay all produced blocks" tReplay = "Replay all produced blocks"
proc gauss(r: var Rand; mu = 0.0; sigma = 1.0): float =
# TODO This is present in Nim 1.4
const K = sqrt(2 / E)
var
a = 0.0
b = 0.0
while true:
a = rand(r, 1.0)
b = (2.0 * rand(r, 1.0) - 1.0) * K
if b * b <= -4.0 * a * a * ln(a): break
result = mu + sigma * (b / a)
# TODO confutils is an impenetrable black box. how can a help text be added here? # TODO confutils is an impenetrable black box. how can a help text be added here?
cli do(slots = SLOTS_PER_EPOCH * 6, cli do(slots = SLOTS_PER_EPOCH * 6,
validators = SLOTS_PER_EPOCH * 200, # One per shard is minimum validators = SLOTS_PER_EPOCH * 200, # One per shard is minimum
@ -44,9 +56,10 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
blockRatio {.desc: "ratio of slots with blocks"} = 1.0, blockRatio {.desc: "ratio of slots with blocks"} = 1.0,
replay = true): replay = true):
let let
state = loadGenesis(validators, true) (state, depositContractSnapshot) = loadGenesis(validators, false)
genesisBlock = get_initial_beacon_block(state[].data) genesisBlock = get_initial_beacon_block(state[].data)
runtimePreset = defaultRuntimePreset runtimePreset = defaultRuntimePreset
genesisTime = float state[].data.genesis_time
echo "Starting simulation..." echo "Starting simulation..."
@ -54,15 +67,25 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
defer: db.close() defer: db.close()
ChainDAGRef.preInit(db, state[].data, state[].data, genesisBlock) ChainDAGRef.preInit(db, state[].data, state[].data, genesisBlock)
putInitialDepositContractSnapshot(db, depositContractSnapshot)
var var
chainDag = init(ChainDAGRef, runtimePreset, db) chainDag = ChainDAGRef.init(runtimePreset, db)
eth1Chain = Eth1Chain.init(runtimePreset, db)
merkleizer = depositContractSnapshot.createMerkleizer
quarantine = QuarantineRef() quarantine = QuarantineRef()
attPool = AttestationPool.init(chainDag, quarantine) attPool = AttestationPool.init(chainDag, quarantine)
timers: array[Timers, RunningStat] timers: array[Timers, RunningStat]
attesters: RunningStat attesters: RunningStat
r = initRand(1) r = initRand(1)
eth1Chain.addBlock Eth1Block(
number: Eth1BlockNumber 1,
timestamp: Eth1BlockTimestamp genesisTime,
voteData: Eth1Data(
deposit_root: merkleizer.getDepositsRoot,
deposit_count: merkleizer.getChunkCount))
let replayState = assignClone(chainDag.headState) let replayState = assignClone(chainDag.headState)
proc handleAttestations(slot: Slot) = proc handleAttestations(slot: Slot) =
@ -105,20 +128,23 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
chainDag.withState(chainDag.tmpState, head.atSlot(slot)): chainDag.withState(chainDag.tmpState, head.atSlot(slot)):
let let
finalizedEpochRef = chainDag.getFinalizedEpochRef()
proposerIdx = get_beacon_proposer_index(state, cache).get() proposerIdx = get_beacon_proposer_index(state, cache).get()
privKey = hackPrivKey(state.validators[proposerIdx]) privKey = hackPrivKey(state.validators[proposerIdx])
eth1data = get_eth1data_stub( eth1ProposalData = eth1Chain.getBlockProposalData(
state.eth1_deposit_index, slot.compute_epoch_at_slot()) state,
finalizedEpochRef.eth1_data,
finalizedEpochRef.eth1_deposit_index)
message = makeBeaconBlock( message = makeBeaconBlock(
runtimePreset, runtimePreset,
hashedState, hashedState,
proposerIdx, proposerIdx,
head.root, head.root,
privKey.genRandaoReveal(state.fork, state.genesis_validators_root, slot), privKey.genRandaoReveal(state.fork, state.genesis_validators_root, slot),
eth1data, eth1ProposalData.vote,
default(GraffitiBytes), default(GraffitiBytes),
attPool.getAttestationsForBlock(state, cache), attPool.getAttestationsForBlock(state, cache),
@[], eth1ProposalData.deposits,
@[], @[],
@[], @[],
@[], @[],
@ -148,12 +174,42 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
blck() = added[] blck() = added[]
chainDag.updateHead(added[], quarantine) chainDag.updateHead(added[], quarantine)
var
lastEth1BlockAt = genesisTime
eth1BlockNum = 1000
for i in 0..<slots: for i in 0..<slots:
let let
slot = Slot(i + 1) slot = Slot(i + 1)
t = t =
if slot.isEpoch: tEpoch if slot.isEpoch: tEpoch
else: tBlock else: tBlock
now = genesisTime + float(slot * SECONDS_PER_SLOT)
while true:
let nextBlockTime = lastEth1BlockAt +
max(1.0, gauss(r, float SECONDS_PER_ETH1_BLOCK, 3.0))
if nextBlockTime > now:
break
inc eth1BlockNum
var eth1Block = Eth1Block(
number: Eth1BlockNumber eth1BlockNum,
timestamp: Eth1BlockTimestamp nextBlockTime,
voteData: Eth1Data(
block_hash: makeFakeHash(eth1BlockNum)))
let newDeposits = int clamp(gauss(r, 5.0, 8.0), 0.0, 1000.0)
for i in 0 ..< newDeposits:
let d = makeDeposit(merkleizer.getChunkCount.int, {skipBLSValidation})
eth1Block.deposits.add d
merkleizer.addChunk hash_tree_root(d).data
eth1Block.voteData.deposit_root = merkleizer.getDepositsRoot
eth1Block.voteData.deposit_count = merkleizer.getChunkCount
eth1Chain.addBlock eth1Block
lastEth1BlockAt = nextBlockTime
if blockRatio > 0.0: if blockRatio > 0.0:
withTimer(timers[t]): withTimer(timers[t]):

39
research/simutils.nim
View File

@ -1,7 +1,7 @@
import import
stats, os, strformat, times, stats, os, strformat, times,
../tests/[testblockutil], ../tests/[testblockutil],
../beacon_chain/[extras], ../beacon_chain/[extras, eth1_monitor, beacon_chain_db],
../beacon_chain/ssz/[merkleization, ssz_serialization], ../beacon_chain/ssz/[merkleization, ssz_serialization],
../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, presets] ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, presets]
@ -40,11 +40,17 @@ func verifyConsensus*(state: BeaconState, attesterRatio: auto) =
if current_epoch >= 4: if current_epoch >= 4:
doAssert state.finalized_checkpoint.epoch + 2 >= current_epoch doAssert state.finalized_checkpoint.epoch + 2 >= current_epoch
proc loadGenesis*(validators: Natural, validate: bool): ref HashedBeaconState = proc loadGenesis*(validators: Natural, validate: bool):
let fn = &"genesim_{const_preset}_{validators}_{SPEC_VERSION}.ssz" (ref HashedBeaconState, DepositContractSnapshot) =
let res = (ref HashedBeaconState)() let
if fileExists(fn): genesisFn =
res.data = SSZ.loadFile(fn, BeaconState) &"genesis_{const_preset}_{validators}_{SPEC_VERSION}.ssz"
contractSnapshotFn =
&"deposit_contract_snapshot_{const_preset}_{validators}_{SPEC_VERSION}.ssz"
res = (ref HashedBeaconState)()
if fileExists(genesisFn) and fileExists(contractSnapshotFn):
res.data = SSZ.loadFile(genesisFn, BeaconState)
res.root = hash_tree_root(res.data) res.root = hash_tree_root(res.data)
if res.data.slot != GENESIS_SLOT: if res.data.slot != GENESIS_SLOT:
echo "Can only start from genesis state" echo "Can only start from genesis state"
@ -53,9 +59,13 @@ proc loadGenesis*(validators: Natural, validate: bool): ref HashedBeaconState =
if res.data.validators.len != validators: if res.data.validators.len != validators:
echo &"Supplied genesis file has {res.data.validators.len} validators, while {validators} where requested, running anyway" echo &"Supplied genesis file has {res.data.validators.len} validators, while {validators} where requested, running anyway"
echo &"Loaded {fn}..." echo &"Loaded {genesisFn}..."
# TODO check that the private keys are interop keys # TODO check that the private keys are interop keys
res
let contractSnapshot = SSZ.loadFile(contractSnapshotFn,
DepositContractSnapshot)
(res, contractSnapshot)
else: else:
echo "Genesis file not found, making one up (use nimbus_beacon_node createTestnet to make one)" echo "Genesis file not found, making one up (use nimbus_beacon_node createTestnet to make one)"
@ -65,6 +75,11 @@ proc loadGenesis*(validators: Natural, validate: bool): ref HashedBeaconState =
deposits = makeInitialDeposits(validators.uint64, flags) deposits = makeInitialDeposits(validators.uint64, flags)
echo "Generating Genesis..." echo "Generating Genesis..."
var merkleizer = init DepositsMerkleizer
for d in deposits:
merkleizer.addChunk hash_tree_root(d).data
let contractSnapshot = DepositContractSnapshot(
depositContractState: merkleizer.toDepositContractState)
res.data = initialize_beacon_state_from_eth1( res.data = initialize_beacon_state_from_eth1(
defaultRuntimePreset, defaultRuntimePreset,
@ -75,10 +90,12 @@ proc loadGenesis*(validators: Natural, validate: bool): ref HashedBeaconState =
res.root = hash_tree_root(res.data) res.root = hash_tree_root(res.data)
echo &"Saving to {fn}..." echo &"Saving to {genesisFn}..."
SSZ.saveFile(fn, res.data) SSZ.saveFile(genesisFn, res.data)
echo &"Saving to {contractSnapshotFn}..."
SSZ.saveFile(contractSnapshotFn, contractSnapshot)
res (res, contractSnapshot)
proc printTimers*[Timers: enum]( proc printTimers*[Timers: enum](
validate: bool, validate: bool,

2
research/state_sim.nim
View File

@ -42,7 +42,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
validate = true): validate = true):
let let
flags = if validate: {} else: {skipBlsValidation} flags = if validate: {} else: {skipBlsValidation}
state = loadGenesis(validators, validate) (state, depositContractState) = loadGenesis(validators, validate)
genesisBlock = get_initial_beacon_block(state.data) genesisBlock = get_initial_beacon_block(state.data)
echo "Starting simulation..." echo "Starting simulation..."

7
tests/official/test_fixture_ssz_consensus_objects.nim
View File

@ -33,6 +33,13 @@ type
# Some have a signing_root field # Some have a signing_root field
signing_root: string signing_root: string
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/validator.md#eth1block
Eth1Block* = object
timestamp*: uint64
deposit_root*: Eth2Digest
deposit_count*: uint64
# All other eth1 block fields
# Make signing root optional # Make signing root optional
setDefaultValue(SSZHashTreeRoot, signing_root, "") setDefaultValue(SSZHashTreeRoot, signing_root, "")

4
tests/testblockutil.nim
View File

@ -19,7 +19,7 @@ func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey =
var bytes = uint64(i + 1000).toBytesLE() var bytes = uint64(i + 1000).toBytesLE()
copyMem(addr result, addr bytes[0], sizeof(bytes)) copyMem(addr result, addr bytes[0], sizeof(bytes))
func makeFakeHash(i: int): Eth2Digest = func makeFakeHash*(i: int): Eth2Digest =
var bytes = uint64(i).toBytesLE() var bytes = uint64(i).toBytesLE()
static: doAssert sizeof(bytes) <= sizeof(result.data) static: doAssert sizeof(bytes) <= sizeof(result.data)
copyMem(addr result.data[0], addr bytes[0], sizeof(bytes)) copyMem(addr result.data[0], addr bytes[0], sizeof(bytes))
@ -34,7 +34,7 @@ func hackPrivKey*(v: Validator): ValidatorPrivKey =
let i = int(uint64.fromBytesLE(bytes)) let i = int(uint64.fromBytesLE(bytes))
makeFakeValidatorPrivKey(i) makeFakeValidatorPrivKey(i)
func makeDeposit(i: int, flags: UpdateFlags): DepositData = func makeDeposit*(i: int, flags: UpdateFlags = {}): DepositData =
## Ugly hack for now: we stick the private key in withdrawal_credentials ## Ugly hack for now: we stick the private key in withdrawal_credentials
## which means we can repro private key and randao reveal from this data, ## which means we can repro private key and randao reveal from this data,
## for testing :) ## for testing :)