mirror of
https://github.com/status-im/nimbus-eth1.git
synced 2025-01-13 22:04:52 +00:00
a0d0e35a70
* Renamed source file clique_utils => clique_helpers
why:
New name is more in line with other modules where local libraries
are named similarly.
* re-implemented PoA verification module as clique_verify.nim
details:
The verification code was ported from the go sources and provisionally
stored in the clique_misc.nim source file.
todo:
Bring it to life.
* re-design Snapshot descriptor as: ref object
why:
Avoids some copying descriptor objects
details:
The snapshot management in clique_snapshot.nim has been cleaned up.
todo:
There is a lot of unnecessary copying & sub-list manipulation of
seq[BlockHeader] lists which needs to be simplified by managing
index intervals.
* optimised sequence handling for Clique/PoA
why:
To much ado about nothing
details:
* Working with shallow sequences inside PoA processing avoids
unnecessary copying.
* Using degenerate lists in the cliqueVerify() batch where only the
parent (and no other ancestor) is needed.
todo:
Expose only functions that are needed, shallow sequences should be
handles with care.
* fix var-parameter function argument
* Activate PoA engine -- currently proof of concept
details:
PoA engine is activated with newChain(extraValidation = true) applied
to a PoA network.
status and todo:
The extraValidation flag on the Chain object can be set at a later
state which allows to pre-load parts of the block chain without
verification. Setting it later will only go back the block chain to
the latest epoch checkpoint. This is inherent to the Clique protocol,
needs testing though.
PoA engine works in fine weather mode on Goerli replay. With the
canonical eip-225 tests, there are quite a few fringe conditions
that fail. These can easily fudged over to make things work but need
some more work to understand and correct properly.
* Make the last offending verification header available
why:
Makes some fringe case tests work.
details:
Within a failed transaction comprising several blocks, this
feature help to identify the offending block if there was a
PoA verification error.
* Make PoA header verifier store the final snapshot
why:
The last snapshot needed by the verifier is the one of the parent but
the list of authorised signer is derived from the current snapshot. So
updating to the latest snapshot provides the latest signers list.
details:
Also, PoA processing has been implemented as transaction in
persistBlocks() with Clique state rollback.
Clique tests succeed now.
* Avoiding double yields in iterator => replaced by template
why:
Tanks to Andri who observed it (see #762)
* Calibrate logging interval and fix logging event detection
why:
Logging interval as copied from Go implementation was too large and
needed re-calibration. Elapsed time calculation was bonkers, negative
the wrong way round.
483 lines
17 KiB
Nim
483 lines
17 KiB
Nim
# Nimbus
|
|
# Copyright (c) 2018-2019 Status Research & Development GmbH
|
|
# Licensed under either of
|
|
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
|
|
# http://www.apache.org/licenses/LICENSE-2.0)
|
|
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
|
|
# http://opensource.org/licenses/MIT)
|
|
# at your option. This file may not be copied, modified, or distributed except
|
|
# according to those terms.
|
|
|
|
import
|
|
std/[random, sequtils, strformat, strutils, tables, times],
|
|
../../nimbus/[config, chain_config, constants, genesis, utils],
|
|
../../nimbus/db/db_chain,
|
|
../../nimbus/p2p/[chain,
|
|
clique,
|
|
clique/clique_helpers,
|
|
clique/clique_snapshot,
|
|
clique/snapshot/snapshot_desc],
|
|
./voter_samples as vs,
|
|
eth/[common, keys, p2p, rlp, trie/db],
|
|
ethash,
|
|
secp256k1_abi,
|
|
stew/objects
|
|
|
|
export
|
|
vs, snapshot_desc
|
|
|
|
const
|
|
prngSeed = 42
|
|
|
|
type
|
|
XSealKey = array[EXTRA_SEAL,byte]
|
|
XSealValue = object
|
|
blockNumber: uint64
|
|
account: string
|
|
|
|
TesterPool* = ref object ## Pool to maintain currently active tester accounts,
|
|
## mapped from textual names used in the tests below
|
|
## to actual Ethereum private keys capable of signing
|
|
## transactions.
|
|
prng: Rand
|
|
accounts: Table[string,PrivateKey] ## accounts table
|
|
boot: CustomGenesis ## imported Genesis configuration
|
|
batch: seq[seq[BlockHeader]] ## collect header chains
|
|
chain: Chain
|
|
|
|
names: Table[EthAddress,string] ## reverse lookup for debugging
|
|
xSeals: Table[XSealKey,XSealValue] ## collect signatures for debugging
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private Helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc getBlockHeader(ap: TesterPool; number: BlockNumber): BlockHeader =
|
|
## Shortcut => db/db_chain.getBlockHeader()
|
|
doAssert ap.chain.clique.db.getBlockHeader(number, result)
|
|
|
|
proc getBlockHeader(ap: TesterPool; hash: Hash256): BlockHeader =
|
|
## Shortcut => db/db_chain.getBlockHeader()
|
|
doAssert ap.chain.clique.db.getBlockHeader(hash, result)
|
|
|
|
proc isZero(a: openArray[byte]): bool =
|
|
result = true
|
|
for w in a:
|
|
if w != 0:
|
|
return false
|
|
|
|
proc rand(ap: TesterPool): byte =
|
|
ap.prng.rand(255).byte
|
|
|
|
proc newPrivateKey(ap: TesterPool): PrivateKey =
|
|
## Roughly modelled after `random(PrivateKey,getRng()[])` with
|
|
## non-secure but reproducible PRNG
|
|
var data{.noinit.}: array[SkRawSecretKeySize,byte]
|
|
for n in 0 ..< data.len:
|
|
data[n] = ap.rand
|
|
# verify generated key, see keys.random(PrivateKey) from eth/keys.nim
|
|
var dataPtr0 = cast[ptr cuchar](unsafeAddr data[0])
|
|
doAssert secp256k1_ec_seckey_verify(
|
|
secp256k1_context_no_precomp, dataPtr0) == 1
|
|
# Convert to PrivateKey
|
|
PrivateKey.fromRaw(data).value
|
|
|
|
proc privateKey(ap: TesterPool; account: string): PrivateKey =
|
|
## Return private key for given tester `account`
|
|
if account != "":
|
|
if account in ap.accounts:
|
|
result = ap.accounts[account]
|
|
else:
|
|
result = ap.newPrivateKey
|
|
ap.accounts[account] = result
|
|
let address = result.toPublicKey.toCanonicalAddress
|
|
ap.names[address] = account
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private pretty printer call backs
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc findName(ap: TesterPool; address: EthAddress): string =
|
|
## Find name for a particular address
|
|
if address notin ap.names:
|
|
ap.names[address] = &"X{ap.names.len+1}"
|
|
ap.names[address]
|
|
|
|
proc findSignature(ap: TesterPool; sig: openArray[byte]): XSealValue =
|
|
## Find a previusly registered signature
|
|
if sig.len == XSealKey.len:
|
|
let key = toArray(XSealKey.len,sig)
|
|
if key in ap.xSeals:
|
|
result = ap.xSeals[key]
|
|
|
|
proc ppNonce(ap: TesterPool; v: BlockNonce): string =
|
|
## Pretty print nonce
|
|
if v == NONCE_AUTH:
|
|
"AUTH"
|
|
elif v == NONCE_DROP:
|
|
"DROP"
|
|
else:
|
|
&"0x{v.toHex}"
|
|
|
|
proc ppAddress(ap: TesterPool; v: EthAddress): string =
|
|
## Pretty print address
|
|
if v.isZero:
|
|
result = "@0"
|
|
else:
|
|
let a = ap.findName(v)
|
|
if a == "":
|
|
result = &"@{v}"
|
|
else:
|
|
result = &"@{a}"
|
|
|
|
proc ppExtraData(ap: TesterPool; v: Blob): string =
|
|
## Visualise `extraData` field
|
|
|
|
if v.len < EXTRA_VANITY + EXTRA_SEAL or
|
|
((v.len - (EXTRA_VANITY + EXTRA_SEAL)) mod EthAddress.len) != 0:
|
|
result = &"0x{v.toHex}[{v.len}]"
|
|
else:
|
|
var data = v
|
|
#
|
|
# extra vanity prefix
|
|
let vanity = data[0 ..< EXTRA_VANITY]
|
|
data = data[EXTRA_VANITY ..< data.len]
|
|
result = if vanity.isZero: "0u256+" else: &"{vanity.toHex}+"
|
|
#
|
|
# list of addresses
|
|
if EthAddress.len + EXTRA_SEAL <= data.len:
|
|
var glue = "["
|
|
while EthAddress.len + EXTRA_SEAL <= data.len:
|
|
let address = toArray(EthAddress.len,data[0 ..< EthAddress.len])
|
|
data = data[EthAddress.len ..< data.len]
|
|
result &= &"{glue}{ap.ppAddress(address)}"
|
|
glue = ","
|
|
result &= "]+"
|
|
#
|
|
# signature
|
|
let val = ap.findSignature(data)
|
|
if val.account != "":
|
|
result &= &"<#{val.blockNumber},{val.account}>"
|
|
elif data.isZero:
|
|
result &= &"<0>"
|
|
else:
|
|
let sig = SkSignature.fromRaw(data)
|
|
if sig.isOk:
|
|
result &= &"<{sig.value.toHex}>"
|
|
else:
|
|
result &= &"0x{data.toHex}[{data.len}]"
|
|
|
|
proc ppBlockHeader(ap: TesterPool; v: BlockHeader; delim: string): string =
|
|
## Pretty print block header
|
|
let sep = if 0 < delim.len: delim else: ";"
|
|
&"(blockNumber=#{v.blockNumber.truncate(uint64)}" &
|
|
&"{sep}parentHash={v.parentHash}" &
|
|
&"{sep}selfHash={v.hash}" &
|
|
&"{sep}stateRoot={v.stateRoot}" &
|
|
&"{sep}coinbase={ap.ppAddress(v.coinbase)}" &
|
|
&"{sep}nonce={ap.ppNonce(v.nonce)}" &
|
|
&"{sep}extraData={ap.ppExtraData(v.extraData)})"
|
|
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Private: Constructor helpers
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc initPrettyPrinters(pp: var PrettyPrinters; ap: TesterPool) =
|
|
pp.nonce = proc(v:BlockNonce): string = ap.ppNonce(v)
|
|
pp.address = proc(v:EthAddress): string = ap.ppAddress(v)
|
|
pp.extraData = proc(v:Blob): string = ap.ppExtraData(v)
|
|
pp.blockHeader = proc(v:BlockHeader; d:string): string = ap.ppBlockHeader(v,d)
|
|
|
|
proc resetChainDb(ap: TesterPool; extraData: Blob; debug = false) =
|
|
## Setup new block chain with bespoke genesis
|
|
ap.chain = BaseChainDB(
|
|
db: newMemoryDb(),
|
|
config: ap.boot.config).newChain(extraValidation = true)
|
|
ap.chain.clique.db.populateProgress
|
|
# new genesis block
|
|
var g = ap.boot.genesis
|
|
if 0 < extraData.len:
|
|
g.extraData = extraData
|
|
g.commit(ap.chain.clique.db)
|
|
# fine tune Clique descriptor
|
|
ap.chain.clique.cfg.debug = debug
|
|
ap.chain.clique.cfg.prettyPrint.initPrettyPrinters(ap)
|
|
|
|
proc initTesterPool(ap: TesterPool): TesterPool {.discardable.} =
|
|
result = ap
|
|
result.prng = initRand(prngSeed)
|
|
result.batch = @[newSeq[BlockHeader]()]
|
|
result.accounts = initTable[string,PrivateKey]()
|
|
result.xSeals = initTable[XSealKey,XSealValue]()
|
|
result.names = initTable[EthAddress,string]()
|
|
result.resetChainDb(@[])
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public: pretty printer support
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc getPrettyPrinters*(t: TesterPool): var PrettyPrinters =
|
|
## Mixin for pretty printers, see `clique/clique_cfg.pp()`
|
|
t.chain.clique.cfg.prettyPrint
|
|
|
|
proc say*(t: TesterPool; v: varargs[string,`$`]) =
|
|
if t.chain.clique.cfg.debug:
|
|
stderr.write v.join & "\n"
|
|
|
|
proc sayHeaderChain*(ap: TesterPool; indent = 0): TesterPool {.discardable.} =
|
|
result = ap
|
|
let pfx = ' '.repeat(indent)
|
|
var top = if 0 < ap.batch[^1].len: ap.batch[^1][^1]
|
|
else: ap.getBlockHeader(0.u256)
|
|
ap.say pfx, " top header: " & ap.pp(top, 16+indent)
|
|
while not top.blockNumber.isZero:
|
|
top = ap.getBlockHeader(top.parentHash)
|
|
ap.say pfx, "parent header: " & ap.pp(top, 16+indent)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public: Constructor
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc newVoterPool*(networkId = GoerliNet): TesterPool =
|
|
TesterPool(
|
|
boot: CustomGenesis(
|
|
genesis: defaultGenesisBlockForNetwork(networkId),
|
|
config: chainConfig(networkId))).initTesterPool
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public: getter
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc chain*(ap: TesterPool): auto {.inline.} =
|
|
## Getter
|
|
ap.chain
|
|
|
|
proc clique*(ap: TesterPool): auto {.inline.} =
|
|
## Getter
|
|
ap.chain.clique
|
|
|
|
proc db*(ap: TesterPool): auto {.inline.} =
|
|
## Getter
|
|
ap.clique.db
|
|
|
|
proc debug*(ap: TesterPool): auto {.inline.} =
|
|
## Getter
|
|
ap.clique.cfg.debug
|
|
|
|
proc cliqueSigners*(ap: TesterPool; lastOk = false): auto {.inline.} =
|
|
## Getter
|
|
ap.clique.cliqueSigners
|
|
|
|
proc snapshot*(ap: TesterPool): auto {.inline.} =
|
|
## Getter
|
|
ap.clique.snapshot
|
|
|
|
proc failed*(ap: TesterPool): CliqueFailed {.inline.} =
|
|
## Getter
|
|
ap.clique.failed
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public: setter
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc `debug=`*(ap: TesterPool; debug: bool) {.inline,} =
|
|
## Set debugging mode on/off
|
|
ap.clique.cfg.debug = debug
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public functions
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# clique/snapshot_test.go(62): func (ap *testerAccountPool) address(account [..]
|
|
proc address*(ap: TesterPool; account: string): EthAddress =
|
|
## retrieves the Ethereum address of a tester account by label, creating
|
|
## a new account if no previous one exists yet.
|
|
if account != "":
|
|
result = ap.privateKey(account).toPublicKey.toCanonicalAddress
|
|
|
|
|
|
# clique/snapshot_test.go(49): func (ap *testerAccountPool) [..]
|
|
proc checkpoint*(ap: TesterPool;
|
|
header: var BlockHeader; signers: openArray[string]) =
|
|
## creates a Clique checkpoint signer section from the provided list
|
|
## of authorized signers and embeds it into the provided header.
|
|
header.extraData.setLen(EXTRA_VANITY)
|
|
header.extraData.add signers
|
|
.mapIt(ap.address(it))
|
|
.sorted(EthAscending)
|
|
.mapIt(toSeq(it))
|
|
.concat
|
|
header.extraData.add 0.byte.repeat(EXTRA_SEAL)
|
|
|
|
|
|
# clique/snapshot_test.go(77): func (ap *testerAccountPool) sign(header n[..]
|
|
proc sign*(ap: TesterPool; header: var BlockHeader; signer: string) =
|
|
## sign calculates a Clique digital signature for the given block and embeds
|
|
## it back into the header.
|
|
#
|
|
# Sign the header and embed the signature in extra data
|
|
let
|
|
hashData = header.hashSealHeader.data
|
|
signature = ap.privateKey(signer).sign(SkMessage(hashData)).toRaw
|
|
extraLen = header.extraData.len
|
|
header.extraData.setLen(extraLen - EXTRA_SEAL)
|
|
header.extraData.add signature
|
|
#
|
|
# Register for debugging
|
|
ap.xSeals[signature] = XSealValue(
|
|
blockNumber: header.blockNumber.truncate(uint64),
|
|
account: signer)
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Public: set up & manage voter database
|
|
# ------------------------------------------------------------------------------
|
|
|
|
proc resetVoterChain*(ap: TesterPool; signers: openArray[string];
|
|
epoch = 0): TesterPool {.discardable.} =
|
|
## Reset the batch list for voter headers and update genesis block
|
|
result = ap
|
|
|
|
ap.batch = @[newSeq[BlockHeader]()]
|
|
|
|
# clique/snapshot_test.go(384): signers := make([]common.Address, [..]
|
|
let signers = signers.mapIt(ap.address(it)).sorted(EthAscending)
|
|
|
|
var extraData = 0.byte.repeat(EXTRA_VANITY)
|
|
|
|
# clique/snapshot_test.go(399): for j, signer := range signers {
|
|
for signer in signers:
|
|
extraData.add signer.toSeq
|
|
|
|
# clique/snapshot_test.go(397):
|
|
extraData.add 0.byte.repeat(EXTRA_SEAL)
|
|
|
|
# store modified genesis block and epoch
|
|
ap.resetChainDb(extraData, ap.debug )
|
|
ap.clique.cfg.epoch = epoch
|
|
|
|
|
|
# clique/snapshot_test.go(415): blocks, _ := core.GenerateChain(&config, [..]
|
|
proc appendVoter*(ap: TesterPool;
|
|
voter: TesterVote): TesterPool {.discardable.} =
|
|
## Append a voter header to the block chain batch list
|
|
result = ap
|
|
|
|
doAssert 0 < ap.batch.len # see initTesterPool() and resetVoterChain()
|
|
let parent = if ap.batch[^1].len == 0:
|
|
ap.getBlockHeader(0.u256)
|
|
else:
|
|
ap.batch[^1][^1]
|
|
|
|
var header = BlockHeader(
|
|
parentHash: parent.hash,
|
|
ommersHash: EMPTY_UNCLE_HASH,
|
|
stateRoot: parent.stateRoot,
|
|
timestamp: parent.timestamp + initDuration(seconds = 100),
|
|
txRoot: BLANK_ROOT_HASH,
|
|
receiptRoot: BLANK_ROOT_HASH,
|
|
blockNumber: parent.blockNumber + 1,
|
|
gasLimit: parent.gasLimit,
|
|
#
|
|
# clique/snapshot_test.go(417): gen.SetCoinbase(accounts.address( [..]
|
|
coinbase: ap.address(voter.voted),
|
|
#
|
|
# clique/snapshot_test.go(418): if tt.votes[j].auth {
|
|
nonce: if voter.auth: NONCE_AUTH else: NONCE_DROP,
|
|
#
|
|
# clique/snapshot_test.go(436): header.Difficulty = diffInTurn [..]
|
|
difficulty: DIFF_INTURN, # Ignored, we just need a valid number
|
|
#
|
|
extraData: 0.byte.repeat(EXTRA_VANITY + EXTRA_SEAL))
|
|
|
|
# clique/snapshot_test.go(432): if auths := tt.votes[j].checkpoint; [..]
|
|
if 0 < voter.checkpoint.len:
|
|
doAssert (header.blockNumber mod ap.clique.cfg.epoch) == 0
|
|
ap.checkpoint(header,voter.checkpoint)
|
|
|
|
# Generate the signature, embed it into the header and the block
|
|
ap.sign(header, voter.signer)
|
|
|
|
if voter.newbatch:
|
|
ap.batch.add @[]
|
|
ap.batch[^1].add header
|
|
|
|
|
|
proc appendVoter*(ap: TesterPool;
|
|
voters: openArray[TesterVote]): TesterPool {.discardable.} =
|
|
## Append a list of voter headers to the block chain batch list
|
|
result = ap
|
|
for voter in voters:
|
|
ap.appendVoter(voter)
|
|
|
|
|
|
proc commitVoterChain*(ap: TesterPool; postProcessOk = false;
|
|
stopFaultyHeader = false): TesterPool {.discardable.} =
|
|
## Write the headers from the voter header batch list to the block chain DB.
|
|
##
|
|
## If `postProcessOk` is set, an additional verification step is added at
|
|
## the end of each transaction.
|
|
##
|
|
## if `stopFaultyHeader` is set, the function stopps immediately on error.
|
|
## Otherwise the offending bloch is removed, the rest of the batch is
|
|
## adjusted and applied again repeatedly.
|
|
result = ap
|
|
ap.chain.clique.fakeDiff = true
|
|
|
|
var reChainOk = false
|
|
for n in 0 ..< ap.batch.len:
|
|
block forLoop:
|
|
|
|
var headers = ap.batch[n]
|
|
while true:
|
|
if headers.len == 0:
|
|
break forLoop # continue with for loop
|
|
|
|
ap.say &"*** transaction ({n}) list: [",
|
|
headers.mapIt(&"#{it.blockNumber}").join(", "), "]"
|
|
|
|
# Realign rest of transaction to existing block chain
|
|
if reChainOk:
|
|
var parent = ap.chain.clique.db.getCanonicalHead
|
|
for i in 0 ..< headers.len:
|
|
headers[i].parentHash = parent.hash
|
|
headers[i].blockNumber = parent.blockNumber + 1
|
|
parent = headers[i]
|
|
|
|
# Perform transaction into the block chain
|
|
let bodies = BlockBody().repeat(headers.len)
|
|
if ap.chain.persistBlocks(headers,bodies) == ValidationResult.OK:
|
|
break
|
|
if stopFaultyHeader:
|
|
return
|
|
|
|
# If the offending block is the last one of the last transaction,
|
|
# then there is nothing to do.
|
|
let culprit = headers.filterIt(ap.failed[0] == it.hash)
|
|
doAssert culprit.len == 1
|
|
let number = culprit[0].blockNumber
|
|
if n + 1 == ap.batch.len and number == headers[^1].blockNumber:
|
|
return
|
|
|
|
# Remove offending block and try again for the rest
|
|
ap.say "*** persistBlocks failed, omitting block #", culprit
|
|
let prevLen = headers.len
|
|
headers = headers.filterIt(number != it.blockNumber)
|
|
doAssert headers.len < prevLen
|
|
reChainOk = true
|
|
|
|
if ap.debug:
|
|
ap.say "*** snapshot argument: #", headers[^1].blockNumber
|
|
ap.sayHeaderChain(8)
|
|
when false: # all addresses are typically pp-mappable
|
|
ap.say " address map: ", toSeq(ap.names.pairs)
|
|
.mapIt(&"@{it[1]}:{it[0]}")
|
|
.sorted
|
|
.join("\n" & ' '.repeat(23))
|
|
if postProcessOk:
|
|
discard ap.clique.cliqueSnapshot(headers[^1])
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# End
|
|
# ------------------------------------------------------------------------------
|