From 1de2cc1a7759e27e212839ddd57a50bbe62194d8 Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Fri, 4 Jun 2021 18:20:37 +0100 Subject: [PATCH] Basic tests for Clique PoA/Consensus engine details: test scenario from eip-225 reference implementation, set up unittes2 test framework smoke test for first sample ok (not functional yet) --- nimbus/makefile | 36 ++- nimbus/p2p/clique.nim | 190 +++++++++----- nimbus/p2p/clique/clique_cfg.nim | 129 +++++++++- nimbus/p2p/clique/clique_defs.nim | 63 ++--- nimbus/p2p/clique/clique_poll.nim | 10 +- nimbus/p2p/clique/clique_utils.nim | 56 +++- nimbus/p2p/clique/ec_recover.nim | 9 +- nimbus/p2p/clique/recent_snaps.nim | 17 +- nimbus/p2p/clique/snapshot.nim | 61 ++--- tests/test_clique.nim | 86 +++++++ tests/test_clique/pool.nim | 386 ++++++++++++++++++++++++++++ tests/test_clique/voter_samples.nim | 339 ++++++++++++++++++++++++ 12 files changed, 1196 insertions(+), 186 deletions(-) create mode 100644 tests/test_clique.nim create mode 100644 tests/test_clique/pool.nim create mode 100644 tests/test_clique/voter_samples.nim diff --git a/nimbus/makefile b/nimbus/makefile index 96b101bc0..d64e1e61e 100644 --- a/nimbus/makefile +++ b/nimbus/makefile @@ -13,9 +13,10 @@ endif # Collect document names SFX_FILTER := sed -e 's|^\./||;/\/\./d;/^docs\//d;s/\.[a-z]*$$//' -PNG_FILES := $(shell find . -name '*.png' -print|$(SFX_FILTER)) -MD_FILES := $(shell find . -name '*.md' -print|$(SFX_FILTER)) -EXE_FILES := $(shell find . -name '*.nim' -print|$(SFX_FILTER)) +PNG_FILES := $(shell find -L . -name '*.png' -print|$(SFX_FILTER)) +MD_FILES := $(shell find -L . -name '*.md' -print|$(SFX_FILTER)) +EXE_FILES := $(shell find -L . -name '*.nim' -print|$(SFX_FILTER)) +TXE_FILES := $(shell find -L ../tests -name '*.nim' -print|$(SFX_FILTER)) # Needed for the NIM compiler that comes with this repo NIMBLE_DIR := $(dir $(PWD))/vendor/.nimble @@ -215,7 +216,7 @@ check_vm: .PHONY: clobber distclean clean clean-exe clean-docs -.SILENT: clean-exe +.SILENT: clean-exe clean-test-exe clean-exe: for f in $(EXE_FILES); do \ if [ -f "$$f" ]; then (set -x; rm -f "$$f"); \ @@ -223,6 +224,13 @@ clean-exe: fi ; \ done +clean-test-exe: + for f in $(TXE_FILES); do \ + if [ -f "$$f" ]; then (set -x; rm -f "$$f"); \ + elif [ -f "$$f.out" ]; then (set -x; rm -f "$$f.out"); \ + fi ; \ + done + .SILENT: clean-docs clean-docs: for f in docs/dochack.js docs/nimdoc.out.css; do \ @@ -245,22 +253,32 @@ clean-docs: (set -x; rmdir "$$d" $(MUFFLE)) || true; \ done -.SILENT: clean-bakfiles +.SILENT: clean-bakfiles clean-test-bakfiles clean-bakfiles: for f in $(shell find . -type f \ \( -name '*~' -o -name '*.bak' \) -print); do \ (set -x; rm -f "$$f"); \ done -.SILENT: clean-nimcache +clean-test-bakfiles: + for f in $(shell find . -type f \ + \( -name '*~' -o -name '*.bak' \) -print); do \ + (set -x; rm -f "$$f"); \ + done + +.SILENT: clean-nimcache clean-test-nimcache clean-nimcache: # |while.. is like "|xargs -rn1 rm -rf" but with nicer log message find . -name 'nimcache' -type d -print 2>/dev/null | \ while read d; do (set -x; rm -rf "$$d"); done -clean:: clean-exe -clean:: clean-bakfiles -clean:: clean-nimcache +clean-test-nimcache: + find ../tests -name 'nimcache' -type d -print 2>/dev/null | \ + while read d; do (set -x; rm -rf "$$d"); done + +clean:: clean-exe clean-test-exe +clean:: clean-bakfiles clean-test-bakfiles +clean:: clean-nimcache clean-test-nimcache distclean:: clean distclean:: clean-docs diff --git a/nimbus/p2p/clique.nim b/nimbus/p2p/clique.nim index 0564095ff..fb14e5b67 100644 --- a/nimbus/p2p/clique.nim +++ b/nimbus/p2p/clique.nim @@ -19,13 +19,13 @@ ## import - ../db/db_chain, + ../constants, + ../db/[db_chain, state_db], ../utils, ./clique/[clique_cfg, clique_defs, clique_utils, ec_recover, recent_snaps], chronicles, chronos, eth/[common, keys, rlp], - # ethash, nimcrypto, random, sequtils, @@ -33,6 +33,10 @@ import tables, times +export + clique_cfg, + clique_defs + type # clique/clique.go(142): type SignerFn func(signer [..] CliqueSignerFn* = ## Hashes and signs the data to be signed by @@ -80,10 +84,10 @@ template doExclusively(c: var Clique; action: untyped) = # ------------------------------------------------------------------------------ # clique/clique.go(145): func ecrecover(header [..] -proc ecrecover(header: BlockHeader; - sigcache: var EcRecover): Result[EthAddress,CliqueError] = +proc ecrecover(c: var Clique; + header: BlockHeader): Result[EthAddress,CliqueError] = ## ecrecover extracts the Ethereum account address from a signed header. - sigcache.getEcRecover(header) + c.cfg.signatures.getEcRecover(header) # clique/clique.go(369): func (c *Clique) snapshot(chain [..] @@ -114,7 +118,7 @@ proc verifySeal(c: var Clique; header: BlockHeader; return err(snap.error) # Resolve the authorization key and check against signers - let signer = ecrecover(header,c.cfg.signatures) + let signer = c.ecrecover(header) if signer.isErr: return err(signer.error) @@ -207,7 +211,7 @@ proc verifyCascadingFields(c: var Clique; header: BlockHeader; return c.verifySeal(header, parents) -# clique/clique.go(145): func ecrecover(header [..] +# clique/clique.go(246): func (c *Clique) verifyHeader(chain [..] proc verifyHeader(c: var Clique; header: BlockHeader; parents: openArray[BlockHeader]): CliqueResult = ## Check whether a header conforms to the consensus rules.The caller may @@ -254,7 +258,7 @@ proc verifyHeader(c: var Clique; header: BlockHeader; # Ensure that the block does not contain any uncles which are meaningless # in PoA - if header.ommersHash != UNCLE_HASH: + if header.ommersHash != EMPTY_UNCLE_HASH: return err((errInvalidUncleHash,"")) # Ensure that the block's difficulty is meaningful (may not be correct at @@ -287,53 +291,56 @@ proc calcDifficulty(snap: var Snapshot; signer: EthAddress): DifficultyInt = else: DIFF_NOTURN -# clique/clique.go(730): func encodeSigHeader(w [..] -proc encodeSigHeader(header: BlockHeader): seq[byte] = - ## Cut sigature off `extraData` header field and consider new `baseFee` - ## field for Eip1559. - doAssert EXTRA_SEAL < header.extraData.len - - var rlpHeader = header - rlpHeader.extraData.setLen(header.extraData.len - EXTRA_SEAL) - - rlpHeader.encode1559 - # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ # clique/clique.go(191): func New(config [..] -proc initClique*(c: var Clique; cfg: CliqueCfg) = +proc initClique*(c: var Clique; cfg: CliqueCfg; testMode = false) = ## Initialiser for Clique proof-of-authority consensus engine with the ## initial signers set to the ones provided by the user. c.cfg = cfg c.recents = initRecentSnaps(cfg) c.proposals = initTable[EthAddress,bool]() c.lock = newAsyncLock() + c.fakeDiff = testMode -proc initClique*(cfg: CliqueCfg): Clique = - result.initClique(cfg) +proc initClique*(cfg: CliqueCfg; testMode = false): Clique = + result.initClique(cfg, testMode) # clique/clique.go(212): func (c *Clique) Author(header [..] proc author*(c: var Clique; header: BlockHeader): Result[EthAddress,CliqueError] = - ## Implements consensus.Engine, returning the Ethereum address recovered - ## from the signature in the header's extra-data section. - ecrecover(header, c.cfg.signatures) + ## For the Consensus Engine, `author()` retrieves the Ethereum address of the + ## account that minted the given block, which may be different from the + ## header's coinbase if a consensus engine is based on signatures. + ## + ## This implementation returns the Ethereum address recovered from the + ## signature in the header's extra-data section. + c.ecrecover(header) # clique/clique.go(217): func (c *Clique) VerifyHeader(chain [..] proc verifyHeader*(c: var Clique; header: BlockHeader): CliqueResult = - ## Checks whether a header conforms to the consensus rules. + ## For the Consensus Engine, `verifyHeader()` checks whether a header + ## conforms to the consensus rules of a given engine. Verifying the seal + ## may be done optionally here, or explicitly via the `verifySeal()` method. + ## + ## This implementation checks whether a header conforms to the consensus + ## rules. c.verifyHeader(header, @[]) # clique/clique.go(224): func (c *Clique) VerifyHeader(chain [..] -proc verifyHeader*(c: var Clique; headers: openArray[BlockHeader]): +proc verifyHeaders*(c: var Clique; headers: openArray[BlockHeader]): Future[seq[CliqueResult]] {.async,gcsafe.} = - ## Checks whether a header conforms to the consensus rules. It verifies - ## a batch of headers. If running in the background, the process can be - ## stopped by calling the `stopVerifyHeader()` function. + ## For the Consensus Engine, `verifyHeader()` s similar to VerifyHeader, but + ## verifies a batch of headers concurrently. This method is accompanied + ## by a `stopVerifyHeader()` method that can abort the operations. + ## + ## This implementation checks whether a header conforms to the consensus + ## rules. It verifies a batch of headers. If running in the background, + ## the process can be stopped by calling the `stopVerifyHeader()` function. c.doExclusively: c.stopVHeaderReq = false for n in 0 ..< headers.len: @@ -357,8 +364,11 @@ proc stopVerifyHeader*(c: var Clique): bool {.discardable.} = # clique/clique.go(450): func (c *Clique) VerifyUncles(chain [..] proc verifyUncles*(c: var Clique; ethBlock: EthBlock): CliqueResult = - ## Always returning an error for any uncles as this consensus mechanism - ## doesn't permit uncles. + ## For the Consensus Engine, `verifyUncles()` verifies that the given + ## block's uncles conform to the consensus rules of a given engine. + ## + ## This implementation always returns an error for existing uncles as this + ## consensus mechanism doesn't permit uncles. if 0 < ethBlock.uncles.len: return err((errCliqueUnclesNotAllowed,"")) result = ok() @@ -366,8 +376,12 @@ proc verifyUncles*(c: var Clique; ethBlock: EthBlock): CliqueResult = # clique/clique.go(506): func (c *Clique) Prepare(chain [..] proc prepare*(c: var Clique; header: var BlockHeader): CliqueResult = - ## Peparing all the consensus fields of the header for running the - ## transactions on top. + ## For the Consensus Engine, `prepare()` initializes the consensus fields + ## of a block header according to the rules of a particular engine. The + ## changes are executed inline. + ## + ## This implementation prepares all the consensus fields of the header for + ## running the transactions on top. # If the block isn't a checkpoint, cast a random vote (good enough for now) header.coinbase.reset @@ -378,7 +392,7 @@ proc prepare*(c: var Clique; header: var BlockHeader): CliqueResult = if snap.isErr: return err(snap.error) - if (header.blockNumber mod c.cfg.epoch.u256) != 0: + if (header.blockNumber mod c.cfg.epoch) != 0: c.doExclusively: # Gather all the proposals that make sense voting on var addresses: seq[EthAddress] @@ -397,10 +411,8 @@ proc prepare*(c: var Clique; header: var BlockHeader): CliqueResult = # Ensure the extra data has all its components header.extraData.setLen(EXTRA_VANITY) - - if (header.blockNumber mod c.cfg.epoch.u256) == 0: - for a in snap.value.signers: - header.extraData.add a + if (header.blockNumber mod c.cfg.epoch) == 0: + header.extraData.add snap.value.signers.mapIt(toSeq(it)).concat header.extraData.add 0.byte.repeat(EXTRA_SEAL) # Mix digest is reserved for now, set to empty @@ -419,30 +431,45 @@ proc prepare*(c: var Clique; header: var BlockHeader): CliqueResult = # clique/clique.go(571): func (c *Clique) Finalize(chain [..] -#proc finalize*(c: var Clique; header: BlockHeader; state: StateDB; -# txs: openArray[Transaction]; uncles: openArray[BlockHeader]) = -# ## Ensuring no uncles are set, nor block rewards given. -# -# # No block rewards in PoA, so the state remains as is and uncles are dropped -# header.Root = -# state.intermediateRoot(c.cfg.config.eip158block <= header.BlockNumber) -# header.UncleHash = types.CalcUncleHash(nil) +proc finalize*(c: var Clique; header: BlockHeader; db: AccountStateDB) = + ## For the Consensus Engine, `finalize()` runs any post-transaction state + ## modifications (e.g. block rewards) but does not assemble the block. + ## + ## Note: The block header and state database might be updated to reflect any + ## consensus rules that happen at finalization (e.g. block rewards). + ## + ## Not implemented here, raises `AssertionDefect` + raiseAssert "Not implemented" + # + # ## This implementation ensures no uncles are set, nor block rewards given. + # # No block rewards in PoA, so the state remains as is and uncles are dropped + # let deleteEmptyObjectsOk = c.cfg.config.eip158block <= header.blockNumber + # header.stateRoot = db.intermediateRoot(deleteEmptyObjectsOk) + # header.ommersHash = EMPTY_UNCLE_HASH # clique/clique.go(579): func (c *Clique) FinalizeAndAssemble(chain [..] -#proc finalizeAndAssemble*(c: var Clique; header: BlockHeader; state: StateDB; -# txs: openArray[Transaction]; -# uncles: openArray[BlockHeader]; -# receipts: openArray[Receipts]): -# Result[EthBlock,CliqueError] = -# ## Ensuring no uncles are set, nor block rewards given, and returns the -# ## final block. -# -# # Finalize block -# c.finalize(header, state, txs, uncles) -# -# # Assemble and return the final block for sealing -# return types.NewBlock(header, txs, nil, receipts, -# trie.NewStackTrie(nil)), nil +proc finalizeAndAssemble*(c: var Clique; header: BlockHeader; + db: AccountStateDB; txs: openArray[Transaction]; + receipts: openArray[Receipt]): + Result[EthBlock,CliqueError] = + ## For the Consensus Engine, `finalizeAndAssemble()` runs any + ## post-transaction state modifications (e.g. block rewards) and assembles + ## the final block. + ## + ## Note: The block header and state database might be updated to reflect any + ## consensus rules that happen at finalization (e.g. block rewards). + ## + ## Not implemented here, raises `AssertionDefect` + raiseAssert "Not implemented" + # ## Ensuring no uncles are set, nor block rewards given, and returns the + # ## final block. + # + # # Finalize block + # c.finalize(header, state, txs, uncles) + # + # # Assemble and return the final block for sealing + # return types.NewBlock(header, txs, nil, receipts, + # trie.NewStackTrie(nil)), nil # clique/clique.go(589): func (c *Clique) Authorize(signer [..] @@ -463,22 +490,30 @@ proc cliqueRlp*(header: BlockHeader): seq[byte] = ## otherwise it panics. This is done to avoid accidentally using both forms ## (signature present or not), which could be abused to produce different ##hashes for the same header. - header.encodeSigHeader + header.encodeSealHeader # clique/clique.go(688): func SealHash(header *types.Header) common.Hash { -proc sealHash(header: BlockHeader): Hash256 = - ## SealHash returns the hash of a block prior to it being sealed. - header.encodeSigHeader.keccakHash +proc sealHash*(header: BlockHeader): Hash256 = + ## For the Consensus Engine, `sealHash()` returns the hash of a block prior + ## to it being sealed. + ## + ## This implementation returns the hash of a block prior to it being sealed. + header.hashSealHeader # clique/clique.go(599): func (c *Clique) Seal(chain [..] proc seal*(c: var Clique; ethBlock: EthBlock): Future[Result[EthBlock,CliqueError]] {.async,gcsafe.} = - ## Attempt to create a sealed block using the local signing credentials. If - ## running in the background, the process can be stopped by calling the - ## `stopSeal()` function. - + ## For the Consensus Engine, `seal()` generates a new sealing request for + ## the given input block and pushes the result into the given channel. + ## + ## Note, the method returns immediately and will send the result async. More + ## than one result may also be returned depending on the consensus algorithm. + ## + ## This implementation attempts to create a sealed block using the local + ## signing credentials. If running in the background, the process can be + ## stopped by calling the `stopSeal()` function. c.doExclusively: c.stopSealReq = false var header = ethBlock.header @@ -571,8 +606,10 @@ proc stopSeal*(c: var Clique): bool {.discardable.} = # clique/clique.go(673): func (c *Clique) CalcDifficulty(chain [..] proc calcDifficulty(c: var Clique; parent: BlockHeader): Result[DifficultyInt,CliqueError] = - ## The difficulty adjustment algorithm. It returns the difficulty - ## that a new block should have: + ## For the Consensus Engine, `calcDifficulty()` is the difficulty adjustment + ## algorithm. It returns the difficulty that a new block should have. + ## + ## This implementation returns the difficulty that a new block should have: ## * DIFF_NOTURN(2) if BLOCK_NUMBER % SIGNER_COUNT != SIGNER_INDEX ## * DIFF_INTURN(1) if BLOCK_NUMBER % SIGNER_COUNT == SIGNER_INDEX var snap = c.snapshot(parent.blockNumber, parent.blockHash, @[]) @@ -586,6 +623,17 @@ proc calcDifficulty(c: var Clique; # ## SealHash returns the hash of a block prior to it being sealed. # header.encodeSigHeader.keccakHash +# ------------------------------------------------------------------------------ +# Test interface +# ------------------------------------------------------------------------------ + +proc snapshotInternal*(c: var Clique; number: BlockNumber; hash: Hash256; + parent: openArray[Blockheader]): auto = + c.snapshot(number, hash, parent) + +proc cfgInternal*(c: var Clique): auto = + c.cfg + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/p2p/clique/clique_cfg.nim b/nimbus/p2p/clique/clique_cfg.nim index 292f1617d..d9d2cd164 100644 --- a/nimbus/p2p/clique/clique_cfg.nim +++ b/nimbus/p2p/clique/clique_cfg.nim @@ -22,19 +22,41 @@ import ../../db/db_chain, ./clique_defs, ./ec_recover, + eth/common, + ethash, random, + sequtils, + stint, + strutils, times const prngSeed = 42 type + SimpleTypePP = BlockNonce|EthAddress|Blob|BlockHeader + SeqTypePP = EthAddress|BlockHeader + + PrettyPrinters* = object + nonce*: proc(v: BlockNonce): + string {.gcsafe,raises: [Defect,CatchableError].} + address*: proc(v: EthAddress): + string {.gcsafe,raises: [Defect,CatchableError].} + extraData*: proc(v: Blob): + string {.gcsafe,raises: [Defect,CatchableError].} + blockHeader*: proc(v: BlockHeader; delim: string): + string {.gcsafe,raises: [Defect,CatchableError].} + CliqueCfg* = ref object dbChain*: BaseChainDB signatures*: EcRecover ## Recent block signatures to speed up mining period*: Duration ## time between blocks to enforce - epoch*: uint64 ## Epoch length to reset votes and checkpoint - prng*: Rand ## PRNG state + prng*: Rand ## PRNG state for internal random generator + epoch*: UInt256 ## The number of blocks after which to checkpoint + ## and reset the pending votes.Suggested 30000 for + ## the testnet to remain analogous to the mainnet + ## ethash epoch. + prettyPrint*: PrettyPrinters ## debugging support {.push raises: [Defect,CatchableError].} @@ -42,14 +64,103 @@ type # Public functions # ------------------------------------------------------------------------------ -proc newCliqueCfg*(dbChain: BaseChainDB; - period = BLOCK_PERIOD; epoch = EPOCH_LENGTH): CliqueCfg = +proc newCliqueCfg*(dbChain: BaseChainDB; period = BLOCK_PERIOD; + epoch = 0.u256): CliqueCfg = CliqueCfg( - dbChain: dbChain, - period: period, - epoch: epoch, - signatures: initEcRecover(), - prng: initRand(prngSeed)) + dbChain: dbChain, + period: period, + epoch: if epoch.isZero: EPOCH_LENGTH.u256 else: epoch, + signatures: initEcRecover(), + prng: initRand(prngSeed), + prettyPrint: PrettyPrinters( + nonce: proc(v:BlockNonce): string = $v, + address: proc(v:EthAddress): string = $v, + extraData: proc(v:Blob): string = $v, + blockHeader: proc(v:BlockHeader; delim:string): string = $v)) + +# ------------------------------------------------------------------------------ +# Debugging +# ------------------------------------------------------------------------------ + +proc pp*(p: var PrettyPrinters; v: BlockNonce): string = + ## Pretty print nonce + p.nonce(v) + +proc pp*(p: var PrettyPrinters; v: EthAddress): string = + ## Pretty print address + p.address(v) + +proc pp*(p: var PrettyPrinters; v: openArray[EthAddress]): seq[string] = + ## Pretty print address list + toSeq(v).mapIt(p.pp(it)) + +proc pp*(p: var PrettyPrinters; v: Blob): string = + ## Visualise `extraData` field + p.extraData(v) + +proc pp*(p: var PrettyPrinters; v: BlockHeader; delim: string): string = + ## Pretty print block header + p.blockHeader(v, delim) + +proc pp*(p: var PrettyPrinters; v: BlockHeader; indent = 3): string = + ## Pretty print block header, NL delimited, indented fields + let delim = if 0 < indent: "\n" & ' '.repeat(indent) else: " " + p.pp(v,delim) + +proc pp*(p: var PrettyPrinters; v: openArray[BlockHeader]): seq[string] = + ## Pretty print list of block headers + toSeq(v).mapIt(p.pp(it,",")) + + +proc pp*[T;V: SimpleTypePP](t: T; v: V): string = + ## Generic prtetty printer, requires `getPrettyPrinters()` function: + ## :: + ## proc getPrettyPrinters(t: SomeLocalType): var PrettyPrinters + ## + mixin getPrettyPrinters + t.getPrettyPrinters.pp(v) + +proc pp*[T;V: var SimpleTypePP](t: var T; v: V): string = + ## Generic prtetty printer, requires `getPrettyPrinters()` function: + ## :: + ## proc getPrettyPrinters(t: var SomeLocalType): var PrettyPrinters + ## + mixin getPrettyPrinters + t.getPrettyPrinters.pp(v) + + +proc pp*[T;V: SeqTypePP](t: T; v: openArray[V]): seq[string] = + ## Generic prtetty printer, requires `getPrettyPrinters()` function: + ## :: + ## proc getPrettyPrinters(t: SomeLocalType): var PrettyPrinters + ## + mixin getPrettyPrinters + t.getPrettyPrinters.pp(v) + +proc pp*[T;V: SeqTypePP](t: var T; v: openArray[V]): seq[string] = + ## Generic prtetty printer, requires `getPrettyPrinters()` function: + ## :: + ## proc getPrettyPrinters(t: var SomeLocalType): var PrettyPrinters + ## + mixin getPrettyPrinters + t.getPrettyPrinters.pp(v) + + +proc pp*[T;X: int|string](t: T; v: BlockHeader; sep: X): string = + ## Generic prtetty printer, requires `getPrettyPrinters()` function: + ## :: + ## proc getPrettyPrinters(t: SomeLocalType): var PrettyPrinters + ## + mixin getPrettyPrinters + t.getPrettyPrinters.pp(v,sep) + +proc pp*[T;X: int|string](t: var T; v: BlockHeader; sep: X): string = + ## Generic prtetty printer, requires `getPrettyPrinters()` function: + ## :: + ## proc getPrettyPrinters(t: var SomeLocalType): var PrettyPrinters + ## + mixin getPrettyPrinters + t.getPrettyPrinters.pp(v,sep) # ------------------------------------------------------------------------------ # End diff --git a/nimbus/p2p/clique/clique_defs.nim b/nimbus/p2p/clique/clique_defs.nim index 633fa47c0..46cabb5a2 100644 --- a/nimbus/p2p/clique/clique_defs.nim +++ b/nimbus/p2p/clique/clique_defs.nim @@ -18,16 +18,10 @@ ## `go-ethereum `_ ## -const - # debugging, enable with: nim c -r -d:noisy:3 ... - noisy {.intdefine.}: int = 0 - isMainOk {.used.} = noisy > 2 - import eth/common, - ethash, - nimcrypto, stew/results, + stint, times {.push raises: [].} @@ -54,46 +48,38 @@ const # clique/clique.go(57): var ( [..] const - EPOCH_LENGTH* = ## Number of blocks after which to checkpoint and reset - ## the pending votes.Suggested 30000 for the testnet to - ## remain analogous to the mainnet ethash epoch. - ethash.EPOCH_LENGTH.uint64 - - BLOCK_PERIOD* = ## Minimum difference in seconds between two consecutive - ## block's timestamps. Suggested 15s for the testnet to - ## remain analogous to the mainnet ethash target. + BLOCK_PERIOD* = ## Minimum difference in seconds between two + ## consecutive block's timestamps. Suggested 15s for + ## the testnet to remain analogous to the mainnet + ## ethash target. initDuration(seconds = 15) - EXTRA_VANITY* = ## Fixed number of extra-data prefix bytes reserved for - ## signer vanity. Suggested 32 bytes to retain the current - ## extra-data allowance and/or use. + EXTRA_VANITY* = ## Fixed number of extra-data prefix bytes reserved for + ## signer vanity. Suggested 32 bytes to retain the + ## current extra-data allowance and/or use. 32 - EXTRA_SEAL* = ## Fixed number of extra-data suffix bytes reserved for - ## signer seal. 65 bytes fixed as signatures are based on - ## the standard secp256k1 curve. + EXTRA_SEAL* = ## Fixed number of extra-data suffix bytes reserved for + ## signer seal. 65 bytes fixed as signatures are based + ## on the standard secp256k1 curve. 65 - NONCE_AUTH* = ## Magic nonce number 0xffffffffffffffff to vote on adding a - ## new signer. + NONCE_AUTH* = ## Magic nonce number 0xffffffffffffffff to vote on + ## adding a new signer. 0xffffffffffffffffu64.toBlockNonce - NONCE_DROP* = ## Magic nonce number 0x0000000000000000 to vote on removing - ## a signer. + NONCE_DROP* = ## Magic nonce number 0x0000000000000000 to vote on + ## removing a signer. 0x0000000000000000u64.toBlockNonce - UNCLE_HASH* = ## Always Keccak256(RLP([])) as uncles are meaningless - ## outside of PoW. - rlpHash[seq[BlockHeader]](@[]) - - DIFF_NOTURN* = ## Block score (difficulty) for blocks containing out-of-turn - ## signatures. Suggested 1 since it just needs to be an - ## arbitrary baseline constant. + DIFF_NOTURN* = ## Block score (difficulty) for blocks containing + ## out-of-turn signatures. Suggested 1 since it just + ## needs to be an arbitrary baseline constant. 1.u256 - DIFF_INTURN* = ## Block score (difficulty) for blocks containing in-turn - ## signatures. Suggested 2 to show a slight preference over - ## out-of-turn signatures. + DIFF_INTURN* = ## Block score (difficulty) for blocks containing + ## in-turn signatures. Suggested 2 to show a slight + ## preference over out-of-turn signatures. 2.u256 # ------------------------------------------------------------------------------ @@ -134,6 +120,8 @@ const # clique/clique.go(76): var ( [..] type CliqueErrorType* = enum + noCliqueError = 0 ## Default/reset value + errUnknownBlock = ## is returned when the list of signers is ## requested for a block that is not part of ## the local blockchain. @@ -240,14 +228,15 @@ type # "invalid block number" - # additional errors, manually added - # --------------------------------- + # additional/bespoke errors, manually added + # ----------------------------------------- errZeroBlockNumberRejected = "Block number must not be Zero" errSkSigResult ## eth/keys subsytem error: signature errSkPubKeyResult ## eth/keys subsytem error: public key + errSnapshotLoad ## DB subsytem error errSnapshotStore ## .. errSnapshotClone diff --git a/nimbus/p2p/clique/clique_poll.nim b/nimbus/p2p/clique/clique_poll.nim index d960b453e..257b9581a 100644 --- a/nimbus/p2p/clique/clique_poll.nim +++ b/nimbus/p2p/clique/clique_poll.nim @@ -19,7 +19,7 @@ ## import - algorithm, + ./clique_utils, eth/common, sequtils, tables @@ -64,13 +64,7 @@ proc initCliquePoll*(t: var CliquePoll; signers: openArray[EthAddress]) = proc authSigners*(t: var CliquePoll): seq[EthAddress] = ## Sorted ascending list of authorised signer addresses - result = toSeq(t.authSig.keys) - result.sort do (x, y: EthAddress) -> int: - for n in 0 ..< x.len: - if x[n] < y[n]: - return -1 - elif y[n] < x[n]: - return 1 + toSeq(t.authSig.keys).sorted(EthAscending) proc isAuthSigner*(t: var CliquePoll; address: EthAddress): bool = ## Check whether `address` is an authorised signer diff --git a/nimbus/p2p/clique/clique_utils.nim b/nimbus/p2p/clique/clique_utils.nim index 09aa4d002..76e20811b 100644 --- a/nimbus/p2p/clique/clique_utils.nim +++ b/nimbus/p2p/clique/clique_utils.nim @@ -28,12 +28,20 @@ import ../../utils, ../../vm_types2, ./clique_defs, + algorithm, eth/[common, rlp], stew/results, stint, strformat, times +type + EthSortOrder* = enum + EthDescending = SortOrder.Descending.ord + EthAscending = SortOrder.Ascending.ord + +{.push raises: [Defect,CatchableError].} + # ------------------------------------------------------------------------------ # Private helpers # ------------------------------------------------------------------------------ @@ -73,6 +81,16 @@ proc isZero*[T: EthAddress|Hash256|Duration](a: T): bool = ## `true` if `a` is all zero a == zeroItem(T) +proc sorted*(e: openArray[EthAddress]; order = EthAscending): seq[EthAddress] = + proc eCmp(x, y: EthAddress): int = + for n in 0 ..< x.len: + if x[n] < y[n]: + return -1 + elif y[n] < x[n]: + return 1 + e.sorted(cmp = eCmp, order = order.SortOrder) + + proc cliqueResultErr*(w: CliqueError): CliqueResult = ## Return error result (syntactic sugar) err(w) @@ -137,15 +155,6 @@ proc baseFee*(header: BlockHeader): GasInt = # FIXME: `baseFee` header field not defined before `London` fork 0.GasInt -# clique/clique.go(730): func encodeSigHeader(w [..] -proc encode1559*(header: BlockHeader): seq[byte] = - ## Encode header field and considering new `baseFee` field for Eip1559. - var writer = initRlpWriter() - writer.append(header) - if not header.baseFee.isZero: - writer.append(header.baseFee) - result = writer.finish - # consensus/misc/eip1559.go(55): func CalcBaseFee(config [..] proc calc1599BaseFee*(c: var ChainConfig; parent: BlockHeader): GasInt = ## calculates the basefee of the header. @@ -211,6 +220,35 @@ proc verify1559Header*(c: var ChainConfig; return ok() +# clique/clique.go(730): func encodeSigHeader(w [..] +proc encode1559*(header: BlockHeader): seq[byte] = + ## Encode header field and considering new `baseFee` field for Eip1559. + var writer = initRlpWriter() + writer.append(header) + if not header.baseFee.isZero: + writer.append(header.baseFee) + result = writer.finish + +# ------------------------------------------------------------------------------ +# Seal hash support +# ------------------------------------------------------------------------------ + +# clique/clique.go(730): func encodeSigHeader(w [..] +proc encodeSealHeader*(header: BlockHeader): seq[byte] = + ## Cut sigature off `extraData` header field and consider new `baseFee` + ## field for Eip1559. + doAssert EXTRA_SEAL < header.extraData.len + + var rlpHeader = header + rlpHeader.extraData.setLen(header.extraData.len - EXTRA_SEAL) + + rlpHeader.encode1559 + +# clique/clique.go(688): func SealHash(header *types.Header) common.Hash { +proc hashSealHeader*(header: BlockHeader): Hash256 = + ## Returns the hash of a block prior to it being sealed. + header.encodeSealHeader.keccakHash + # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/nimbus/p2p/clique/ec_recover.nim b/nimbus/p2p/clique/ec_recover.nim index 241ef341d..874fff143 100644 --- a/nimbus/p2p/clique/ec_recover.nim +++ b/nimbus/p2p/clique/ec_recover.nim @@ -22,6 +22,7 @@ import ../../utils, ../../utils/lru_cache, ./clique_defs, + ./clique_utils, eth/[common, keys, rlp], stint @@ -58,14 +59,14 @@ proc initEcRecover*(cache: var EcRecover) {.gcsafe, raises: [Defect].} = # clique/clique.go(153): if len(header.Extra) < extraSeal { if msg.len < EXTRA_SEAL: return err((errMissingSignature,"")) - let signature = Signature.fromRaw( + let sig = Signature.fromRaw( msg.toOpenArray(msg.len - EXTRA_SEAL, msg.high)) - if signature.isErr: - return err((errSkSigResult,$signature.error)) + if sig.isErr: + return err((errSkSigResult,$sig.error)) # Recover the public key from signature and seal hash # clique/clique.go(159): pubkey, err := crypto.Ecrecover( [..] - let pubKey = recover(signature.value, SKMessage(header.hash.data)) + let pubKey = recover(sig.value, SKMessage(header.hashSealHeader.data)) if pubKey.isErr: return err((errSkPubKeyResult,$pubKey.error)) diff --git a/nimbus/p2p/clique/recent_snaps.nim b/nimbus/p2p/clique/recent_snaps.nim index 590628fac..daa4d335d 100644 --- a/nimbus/p2p/clique/recent_snaps.nim +++ b/nimbus/p2p/clique/recent_snaps.nim @@ -30,6 +30,7 @@ import chronicles, eth/[common, keys], nimcrypto, + sequtils, stint export @@ -61,7 +62,7 @@ type {.push raises: [Defect,CatchableError].} logScope: - topics = "clique snap cache" + topics = "clique PoA recent-snaps" # ------------------------------------------------------------------------------ # Private helpers @@ -72,18 +73,18 @@ proc canDiskCheckPointOk(d: RecentDesc): bool = # If we're at the genesis, snapshot the initial state. if d.args.blockNumber.isZero: return true - # Alternatively if we're at a checkpoint block without a parent # (light client CHT), or we have piled up more headers than allowed # to be re-orged (chain reinit from a freezer), consider the # checkpoint trusted and snapshot it. - if (d.args.blockNumber mod d.cfg.epoch.u256) == 0: - if (FULL_IMMUTABILITY_THRESHOLD < d.local.headers.len) or - d.cfg.dbChain.getBlockHeaderResult(d.args.blockNumber - 1).isErr: + if (d.args.blockNumber mod d.cfg.epoch) == 0: + if FULL_IMMUTABILITY_THRESHOLD < d.local.headers.len: + return true + if d.cfg.dbChain.getBlockHeaderResult(d.args.blockNumber - 1).isErr: return true proc isCheckPointOk(number: BlockNumber): bool = - number mod CHECKPOINT_INTERVAL == 0 + (number mod CHECKPOINT_INTERVAL) == 0 # ------------------------------------------------------------------------------ # Private functions @@ -167,10 +168,14 @@ proc initRecentSnaps*(rs: var RecentSnaps; # Previous snapshot found, apply any pending headers on top of it for i in 0 ..< d.local.headers.len div 2: + # Reverse lst order swap(d.local.headers[i], d.local.headers[^(1+i)]) block: # clique/clique.go(434): snap, err := snap.apply(headers) + echo ">>> applySnapshot(", + d.local.headers.mapIt(it.blockNumber.truncate(int)), ")" let rc = snap.applySnapshot(d.local.headers) + echo "<<< applySnapshot() => ", rc.repr if rc.isErr: return err(rc.error) diff --git a/nimbus/p2p/clique/snapshot.nim b/nimbus/p2p/clique/snapshot.nim index c3ae7535c..eee6e9528 100644 --- a/nimbus/p2p/clique/snapshot.nim +++ b/nimbus/p2p/clique/snapshot.nim @@ -58,6 +58,14 @@ type logScope: topics = "clique snapshot" +# ------------------------------------------------------------------------------ +# Pretty printers for debugging +# ------------------------------------------------------------------------------ + +proc getPrettyPrinters(s: var Snapshot): var PrettyPrinters = + ## Mixin for pretty printers + s.cfg.prettyPrint + # ------------------------------------------------------------------------------ # Private functions needed to support RLP conversion # ------------------------------------------------------------------------------ @@ -88,6 +96,8 @@ proc initSnapshot*(s: var Snapshot; cfg: CliqueCfg; s.data.blockHash = hash s.data.recents = initTable[BlockNumber,EthAddress]() s.data.ballot.initCliquePoll(signers) + echo ">>> initSnapshot #", number.truncate(uint64), + " >> ", s.pp(signers), " >> ", s.pp(s.data.ballot.authSigners) proc initSnapshot*(cfg: CliqueCfg; number: BlockNumber; hash: Hash256; signers: openArray[EthAddress]): Snapshot = @@ -132,6 +142,8 @@ proc applySnapshot*(s: var Snapshot; ## Initialises an authorization snapshot `snap` by applying the `headers` ## to the argument snapshot `s`. + echo ">>> applySnapshot ", s.pp(headers) + # Allow passing in no headers for cleaner code if headers.len == 0: return ok() @@ -139,7 +151,8 @@ proc applySnapshot*(s: var Snapshot; # Sanity check that the headers can be applied if headers[0].blockNumber != s.data.blockNumber + 1: return err((errInvalidVotingChain,"")) - for i in 0 ..< headers.len: + # clique/snapshot.go(191): for i := 0; i < len(headers)-1; i++ { + for i in 0 ..< headers.len - 1: if headers[i+1].blockNumber != headers[i].blockNumber+1: return err((errInvalidVotingChain,"")) @@ -152,13 +165,16 @@ proc applySnapshot*(s: var Snapshot; # clique/snapshot.go(206): for i, header := range headers [..] for headersIndex in 0 ..< headers.len: + echo ">>> applySnapshot headersIndex=", headersIndex let # headersIndex => also used for logging at the end of this loop header = headers[headersIndex] number = header.blockNumber + echo "<<< applySnapshot 1" + # Remove any votes on checkpoint blocks - if number mod s.cfg.epoch.u256 == 0: + if (number mod s.cfg.epoch) == 0: s.data.ballot.initCliquePoll # Delete the oldest signer from the recent list to allow it signing again @@ -167,15 +183,21 @@ proc applySnapshot*(s: var Snapshot; if limit <= number: s.data.recents.del(number - limit) + echo "<<< applySnapshot 2" + # Resolve the authorization key and check against signers let signer = ? s.cfg.signatures.getEcRecover(header) + echo "<<< applySnapshot 3 ", s.pp(signer) if not s.data.ballot.isAuthSigner(signer): return err((errUnauthorizedSigner,"")) + echo "<<< applySnapshot 4" for recent in s.data.recents.values: if recent == signer: return err((errRecentlySigned,"")) s.data.recents[number] = signer + echo "<<< applySnapshot 5" + # Header authorized, discard any previous vote from the signer s.data.ballot.delVote(signer = signer, address = header.coinbase) @@ -191,6 +213,8 @@ proc applySnapshot*(s: var Snapshot; blockNumber: number, authorize: authOk) + echo "<<< applySnapshot 6" + # clique/snapshot.go(269): if limit := uint64(len(snap.Signers)/2 [..] if s.data.ballot.authSignersShrunk: # Signer list shrunk, delete any leftover recent caches @@ -198,6 +222,8 @@ proc applySnapshot*(s: var Snapshot; if limit <= number: s.data.recents.del(number - limit) + echo "<<< applySnapshot 7" + # If we're taking too much time (ecrecover), notify the user once a while if logInterval < logged - getTime(): info "Reconstructing voting history", @@ -249,37 +275,6 @@ proc inTurn*(s: var Snapshot; number: BlockNumber, signer: EthAddress): bool = if ascSignersList[offset] == signer: return (number mod ascSignersList.len.u256) == offset.u256 -# ------------------------------------------------------------------------------ -# Debugging/testing -# ------------------------------------------------------------------------------ - -when isMainModule and isMainOK: - - var - cfg = newMemoryDB().newBaseChainDB.newCliqueCfg - ssh, ss1, ss2: Snapshot - key: Hash256 - hdr: BlockHeader - - ssh.init(cfg, 0.u256, key, @[]) - ssh.data.blockNumber = 77.u256 - key = ssh.data.blockHash - - ssh.store.expect("store failed") - echo ">>> ", rlp.encode(ssh.data) - - ss2.init(cfg, 0.u256, key, @[]) - ss2.load(cfg,key).expect("load failed") - - echo ">>> ", rlp.encode(ss2.data) - - doAssert rlp.encode(ssh.data) == rlp.encode(ss2.data) - #discard ss1.data.sigcache.getEcRecover(hdr) - - ss1 = ss2 - echo "ss1.data: ", ss1.data.repr - echo "ss2.data: ", ss2.data.repr - # ------------------------------------------------------------------------------ # End # ------------------------------------------------------------------------------ diff --git a/tests/test_clique.nim b/tests/test_clique.nim new file mode 100644 index 000000000..46cfb5fcc --- /dev/null +++ b/tests/test_clique.nim @@ -0,0 +1,86 @@ +# Nimbus +# Copyright (c) 2018-2019 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + # ../nimbus/p2p/clique, + ../nimbus/utils, + ./test_clique/pool, + eth/[keys], + # sequtils, + stint, + strformat, + # times, + unittest2 + +proc initSnapshot(p: TesterPool; t: TestSpecs; noisy: bool): auto = + + # Assemble a chain of headers from the cast votes + p.resetVoterChain(t.signers) + for voter in t.votes: + p.appendVoter(voter) + p.commitVoterChain + + let topHeader = p.topVoterHeader + p.snapshot(topHeader.blockNumber, topHeader.hash, @[]) + + +proc notUsedYet(p: TesterPool; tt: TestSpecs; noisy: bool) = + discard + #[ + # Verify the final list of signers against the expected ones + signers = make([]common.Address, len(tt.results)) + for j, signer := range tt.results { + signers[j] = accounts.address(signer) + } + for j := 0; j < len(signers); j++ { + for k := j + 1; k < len(signers); k++ { + if bytes.Compare(signers[j][:], signers[k][:]) > 0 { + signers[j], signers[k] = signers[k], signers[j] + } + } + } + result := snap.signers() + if len(result) != len(signers) { + t.Errorf("test %d: signers mismatch: have %x, want %x",i,result,signers) + continue + } + for j := 0; j < len(result); j++ { + if !bytes.Equal(result[j][:], signers[j][:]) { + t.Errorf( + "test %d, signer %d: signer mismatch: have %x, want %x", + i, j, result[j], signers[j]) + } + } + ]# + +# clique/snapshot_test.go(99): func TestClique(t *testing.T) { +proc cliqueMain*(noisy = defined(debug)) = + ## Tests that Clique signer voting is evaluated correctly for various simple + ## and complex scenarios, as well as that a few special corner cases fail + ## correctly. + suite "Clicque PoA": + var pool = newTesterPool() + const maxID = 2 + + # clique/snapshot_test.go(379): for i, tt := range tests { + for tt in voterSamples: + if maxId < tt.id: + echo "Tests stopped" + break + + test &"Snapshots {tt.id}: {tt.info.substr(0,50)}...": + var snap = pool.initSnapshot(tt, noisy) + check snap.isOk + + +when isMainModule: + cliqueMain() + +# End diff --git a/tests/test_clique/pool.nim b/tests/test_clique/pool.nim new file mode 100644 index 000000000..7f1e52fa0 --- /dev/null +++ b/tests/test_clique/pool.nim @@ -0,0 +1,386 @@ +# Nimbus +# Copyright (c) 2018-2019 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + ../../nimbus/config, + ../../nimbus/chain_config, + ../../nimbus/constants, + ../../nimbus/utils, + ../../nimbus/db/db_chain, + ../../nimbus/genesis, + ../../nimbus/p2p/clique, + ../../nimbus/p2p/clique/[clique_defs, clique_utils], + ../../nimbus/utils, + ./voter_samples as vs, + eth/[common, keys, rlp, trie/db], + ethash, + random, + secp256k1_abi, + sequtils, + stew/objects, + strformat, + strutils, + tables, + times + +export + vs + +const + prngSeed = 42 + genesisTemplate = "../customgenesis/berlin2000.json" + +type + XSealKey = array[EXTRA_SEAL,byte] + XSealValue = object + blockNumber: uint64 + account: string + + TesterPool* = ref object ## Pool to maintain currently active tester accounts, + ## mapped from textual names used in the tests below + ## to actual Ethereum private keys capable of signing + ## transactions. + prng: Rand + accounts: Table[string,PrivateKey] ## accounts table + boot: CustomGenesis ## imported Genesis configuration + batch: seq[seq[BlockHeader]] ## collect header chains + engine: Clique + + names: Table[EthAddress,string] ## reverse lookup for debugging + xSeals: Table[XSealKey,XSealValue] ## collect signatures for debugging + +# ------------------------------------------------------------------------------ +# Private Helpers +# ------------------------------------------------------------------------------ + +proc bespokeGenesis(file: string): CustomGenesis = + ## Find genesis block + if file == "": + let networkId = getConfiguration().net.networkId + result.genesis = defaultGenesisBlockForNetwork(networkId) + else: + doAssert genesisTemplate.loadCustomGenesis(result) + result.config.poaEngine = true + +proc chain(ap: TesterPool): BaseChainDB = + ## Getter + ap.engine.cfgInternal.dbChain + +proc isZero(a: openArray[byte]): bool = + result = true + for w in a: + if w != 0: + return false + +proc rand(ap: TesterPool): byte = + ap.prng.rand(255).byte + +proc newPrivateKey(ap: TesterPool): PrivateKey = + ## Roughly modelled after `random(PrivateKey,getRng()[])` with + ## non-secure but reproducible PRNG + var data{.noinit.}: array[SkRawSecretKeySize,byte] + for n in 0 ..< data.len: + data[n] = ap.rand + # verify generated key, see keys.random(PrivateKey) from eth/keys.nim + var dataPtr0 = cast[ptr cuchar](unsafeAddr data[0]) + doAssert secp256k1_ec_seckey_verify( + secp256k1_context_no_precomp, dataPtr0) == 1 + # Convert to PrivateKey + PrivateKey.fromRaw(data).value + +proc privateKey(ap: TesterPool; account: string): PrivateKey = + ## Return private key for given tester `account` + if account != "": + if account in ap.accounts: + result = ap.accounts[account] + else: + result = ap.newPrivateKey + ap.accounts[account] = result + let address = result.toPublicKey.toCanonicalAddress + ap.names[address] = account + +proc resetChainDb(ap: TesterPool; extraData: Blob) = + ## Setup new block chain with bespoke genesis + ap.engine.cfgInternal.dbChain = BaseChainDB( + db: newMemoryDb(), + config: ap.boot.config) + # new genesis block + var g = ap.boot.genesis + if 0 < extraData.len: + g.extraData = extraData + g.commit(ap.engine.cfgInternal.dbChain) + +# ------------------------------------------------------------------------------ +# Private pretty printer call backs +# ------------------------------------------------------------------------------ + +proc findName(ap: TesterPool; address: EthAddress): string = + ## Find name for a particular address + if address in ap.names: + return ap.names[address] + +proc findSignature*(ap: TesterPool; sig: openArray[byte]): XSealValue = + ## Find a previusly registered signature + if sig.len == XSealKey.len: + let key = toArray(XSealKey.len,sig) + if key in ap.xSeals: + result = ap.xSeals[key] + +proc ppNonce(ap: TesterPool; v: BlockNonce): string = + ## Pretty print nonce + if v == NONCE_AUTH: + "AUTH" + elif v == NONCE_DROP: + "DROP" + else: + &"0x{v.toHex}" + +proc ppAddress(ap: TesterPool; v: EthAddress): string = + ## Pretty print address + result = ap.findName(v) + if result == "": + if v.isZero: + result = "@0" + else: + result = $v + +proc ppExtraData(ap: TesterPool; v: Blob): string = + ## Visualise `extraData` field + + if v.len < EXTRA_VANITY + EXTRA_SEAL or + ((v.len - (EXTRA_VANITY + EXTRA_SEAL)) mod EthAddress.len) != 0: + result = &"0x{v.toHex}[{v.len}]" + else: + var data = v + # + # extra vanity prefix + let vanity = data[0 ..< EXTRA_VANITY] + data = data[EXTRA_VANITY ..< data.len] + result = if vanity.isZero: "0u256+" else: &"{vanity.toHex}+" + # + # list of addresses + if EthAddress.len + EXTRA_SEAL <= data.len: + var glue = "[" + while EthAddress.len + EXTRA_SEAL <= data.len: + let address = toArray(EthAddress.len,data[0 ..< EthAddress.len]) + data = data[EthAddress.len ..< data.len] + result &= &"{glue}{ap.ppAddress(address)}" + glue = "," + result &= "]+" + # + # signature + let val = ap.findSignature(data) + if val.account != "": + result &= &"<#{val.blockNumber},{val.account}>" + elif data.isZero: + result &= &"<0>" + else: + let sig = SkSignature.fromRaw(data) + if sig.isOk: + result &= &"<{sig.value.toHex}>" + else: + result &= &"0x{data.toHex}[{data.len}]" + +proc ppBlockHeader(ap: TesterPool; v: BlockHeader; delim: string): string = + ## Pretty print block header + &"(blockNumber=#{v.blockNumber.truncate(uint64)}" & + delim & &"coinbase={ap.ppAddress(v.coinbase)}" & + delim & &"nonce={ap.ppNonce(v.nonce)}" & + delim & &"extraData={ap.ppExtraData(v.extraData)})" + +proc initPrettyPrinters(pp: var PrettyPrinters; ap: TesterPool) = + pp.nonce = proc(v:BlockNonce): string = ap.ppNonce(v) + pp.address = proc(v:EthAddress): string = ap.ppAddress(v) + pp.extraData = proc(v:Blob): string = ap.ppExtraData(v) + pp.blockHeader = proc(v:BlockHeader; d:string): string = ap.ppBlockHeader(v,d) + +# ------------------------------------------------------------------------------ +# Public functions +# ------------------------------------------------------------------------------ + +proc newTesterPool*(epoch = 0.u256; genesisTemplate = ""): TesterPool = + new result + result.boot = genesisTemplate.bespokeGenesis + result.prng = initRand(prngSeed) + result.batch = @[newSeq[BlockHeader]()] + result.accounts = initTable[string,PrivateKey]() + result.xSeals = initTable[XSealKey,XSealValue]() + result.names = initTable[EthAddress,string]() + result.engine = newCliqueCfg( + dbChain = BaseChainDB(), + period = initDuration(seconds = 1), + epoch = epoch) + .initClique(testMode = true) + result.engine.cfgInternal.prettyPrint.initPrettyPrinters(result) + result.resetChainDb(@[]) + +# clique/snapshot_test.go(62): func (ap *testerAccountPool) address(account [..] +proc address*(ap: TesterPool; account: string): EthAddress = + ## retrieves the Ethereum address of a tester account by label, creating + ## a new account if no previous one exists yet. + if account != "": + result = ap.privateKey(account).toPublicKey.toCanonicalAddress + +# clique/snapshot_test.go(49): func (ap *testerAccountPool) [..] +proc checkpoint*(ap: TesterPool; + header: var BlockHeader; signers: openArray[string]) = + ## creates a Clique checkpoint signer section from the provided list + ## of authorized signers and embeds it into the provided header. + header.extraData.setLen(EXTRA_VANITY) + header.extraData.add signers + .mapIt(ap.address(it)) + .sorted(EthAscending) + .mapIt(toSeq(it)) + .concat + header.extraData.add 0.byte.repeat(EXTRA_SEAL) + + +# clique/snapshot_test.go(77): func (ap *testerAccountPool) sign(header n[..] +proc sign*(ap: TesterPool; header: var BlockHeader; signer: string) = + ## sign calculates a Clique digital signature for the given block and embeds + ## it back into the header. + # + # Sign the header and embed the signature in extra data + let + hashData = header.hashSealHeader.data + signature = ap.privateKey(signer).sign(SkMessage(hashData)).toRaw + extraLen = header.extraData.len + header.extraData.setLen(extraLen - EXTRA_SEAL) + header.extraData.add signature + # + # Register for debugging + ap.xSeals[signature] = XSealValue( + blockNumber: header.blockNumber.truncate(uint64), + account: signer) + + +proc snapshot*(ap: TesterPool; number: BlockNumber; hash: Hash256; + parent: openArray[BlockHeader]): auto = + ## Call p2p/clique.snapshotInternal() + ap.engine.snapshotInternal(number, hash, parent) + +# ------------------------------------------------------------------------------ +# Public: set up & manage voter database +# ------------------------------------------------------------------------------ + +proc resetVoterChain*(ap: TesterPool; signers: openArray[string]) = + ## Reset the batch list for voter headers and update genesis block + ap.batch = @[newSeq[BlockHeader]()] + + # clique/snapshot_test.go(384): signers := make([]common.Address, [..] + let signers = signers.mapIt(ap.address(it)).sorted(EthAscending) + + var extraData = 0.byte.repeat(EXTRA_VANITY) + + # clique/snapshot_test.go(399): for j, signer := range signers { + for signer in signers: + extraData.add signer.toSeq + + # clique/snapshot_test.go(397): + extraData.add 0.byte.repeat(EXTRA_SEAL) + + # store modified genesis block + ap.resetChainDb(extraData) + + +# clique/snapshot_test.go(415): blocks, _ := core.GenerateChain(&config, [..] +proc appendVoter*(ap: TesterPool; voter: TesterVote) = + ## Append a voter header to the block chain batch list + doAssert 0 < ap.batch.len # see initTesterPool() and resetVoterChain() + let parent = if ap.batch[^1].len == 0: + ap.chain.getBlockHeader(0.u256) + else: + ap.batch[^1][^1] + + var header = BlockHeader( + parentHash: parent.hash, + ommersHash: EMPTY_UNCLE_HASH, + stateRoot: parent.stateRoot, + timestamp: parent.timestamp + initDuration(seconds = 10), + txRoot: BLANK_ROOT_HASH, + receiptRoot: BLANK_ROOT_HASH, + blockNumber: parent.blockNumber + 1, + gasLimit: parent.gasLimit, + # + # clique/snapshot_test.go(417): gen.SetCoinbase(accounts.address( [..] + coinbase: ap.address(voter.voted), + # + # clique/snapshot_test.go(418): if tt.votes[j].auth { + nonce: if voter.auth: NONCE_AUTH else: NONCE_DROP, + # + # clique/snapshot_test.go(436): header.Difficulty = diffInTurn [..] + difficulty: DIFF_INTURN, # Ignored, we just need a valid number + # + # clique/snapshot_test.go(432): if auths := tt.votes[j].checkpoint; [..] + extraData: 0.byte.repeat( + EXTRA_VANITY + voter.checkpoint.len * EthAddress.len + EXTRA_SEAL)) + + # Generate the signature, embed it into the header and the block + ap.sign(header, voter.signer) + + if voter.newbatch: + ap.batch.add @[] + ap.batch[^1].add header + + +proc commitVoterChain*(ap: TesterPool) = + ## Write the headers from the voter header batch list to the block chain DB + # Create a pristine blockchain with the genesis injected + for headers in ap.batch: + if 0 < headers.len: + doAssert ap.chain.getCanonicalHead.blockNumber < headers[0].blockNumber + + # see p2p/chain.persistBlocks() + ap.chain.highestBlock = headers[^1].blockNumber + let transaction = ap.chain.db.beginTransaction() + for i in 0 ..< headers.len: + let header = headers[i] + + discard ap.chain.persistHeaderToDb(header) + doAssert ap.chain.getCanonicalHead().blockHash == header.blockHash + + discard ap.chain.persistTransactions(header.blockNumber, @[]) + discard ap.chain.persistReceipts(@[]) + ap.chain.currentBlock = header.blockNumber + transaction.commit() + + +proc topVoterHeader*(ap: TesterPool): BlockHeader = + ## Get top header from voter batch list + doAssert 0 < ap.batch.len # see initTesterPool() and resetVoterChain() + if 0 < ap.batch[^1].len: + result = ap.batch[^1][^1] + +proc getPrettyPrinters*(t: TesterPool): var PrettyPrinters = + ## Mixin for pretty printers, see `clique/clique_cfg.pp()` + t.engine.cfgInternal.prettyPrint + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + +#[ +let tt = voterSamples[0] + +var p = newTesterPool() +p.resetVoterChain(tt.signers) +for voter in tt.votes: + p.appendVoter(voter) +p.commitVoterChain + +let topHeader = p.topVoterHeader + +echo "*** adresses: ", toSeq(p.names.pairs).mapIt(&"{it[1]}:{it[0]}").join(", ") +echo " genesis: ", p.pp(p.chain.getBlockHeader(0.u256),15) +echo " topHeader: ", p.pp(topHeader,15) + +var snap = p.snapshot(topHeader.blockNumber, topHeader.hash, @[]) +echo ">>> ", snap +]# diff --git a/tests/test_clique/voter_samples.nim b/tests/test_clique/voter_samples.nim new file mode 100644 index 000000000..0e20c2248 --- /dev/null +++ b/tests/test_clique/voter_samples.nim @@ -0,0 +1,339 @@ +# Nimbus +# Copyright (c) 2018-2019 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# Test cases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-225.md + +import + ../../nimbus/p2p/clique/clique_defs + +type + TesterVote* = object ## VoterBlock represents a single block signed by a + ## particular account, where the account may or may not + ## have cast a Clique vote. + signer*: string ## Account that signed this particular block + voted*: string ## Optional value if the signer voted on + ## adding/removing ## someone + auth*: bool ## Whether the vote was to authorize (or + ## deauthorize) + checkpoint*: seq[string] ## List of authorized signers if this is an epoch + ## block + newbatch*: bool + + TestSpecs* = object ## Define the various voting scenarios to test + id*: int ## Test id + info*: string ## Test description + epoch*: uint64 ## Number of blocks in an epoch (unset = 30000) + signers*: seq[string] ## Initial list of authorized signers in the + ## genesis + votes*: seq[TesterVote] ## Chain of signed blocks, potentially influencing + ## auths + results*: seq[string] ## Final list of authorized signers after all + ## blocks + failure*: CliqueErrorType ## Failure if some block is invalid according to + ## the rules + +const + # Define the various voting scenarios to test + voterSamples* = [ + # clique/snapshot_test.go(108): { + TestSpecs( + id: 1, + info: "Single signer, no votes cast", + signers: @["A"], + votes: @[TesterVote(signer: "A")], + results: @["A"]), + + TestSpecs( + id: 2, + info: "Single signer, voting to add two others (only accept first, "& + "second needs 2 votes)", + signers: @["A"], + votes: @[TesterVote(signer: "A", voted: "B", auth: true), + TesterVote(signer: "B"), + TesterVote(signer: "A", voted: "C", auth: true)], + results: @["A", "B"]), + + TestSpecs( + id: 3, + info: "Two signers, voting to add three others (only accept first " & + "two, third needs 3 votes already)", + signers: @["A", "B"], + votes: @[TesterVote(signer: "A", voted: "C", auth: true), + TesterVote(signer: "B", voted: "C", auth: true), + TesterVote(signer: "A", voted: "D", auth: true), + TesterVote(signer: "B", voted: "D", auth: true), + TesterVote(signer: "C"), + TesterVote(signer: "A", voted: "E", auth: true), + TesterVote(signer: "B", voted: "E", auth: true)], + results: @["A", "B", "C", "D"]), + + TestSpecs( + id: 4, + info: "Single signer, dropping itself (weird, but one less " & + "cornercase by explicitly allowing this)", + signers: @["A"], + votes: @[TesterVote(signer: "A", voted: "A")]), + + TestSpecs( + id: 5, + info: "Two signers, actually needing mutual consent to drop either " & + "of them (not fulfilled)", + signers: @["A", "B"], + votes: @[TesterVote(signer: "A", voted: "B")], + results: @["A", "B"]), + + TestSpecs( + id: 6, + info: "Two signers, actually needing mutual consent to drop either " & + "of them (fulfilled)", + signers: @["A", "B"], + votes: @[TesterVote(signer: "A", voted: "B"), + TesterVote(signer: "B", voted: "B")], + results: @["A"]), + + TestSpecs( + id: 7, + info: "Three signers, two of them deciding to drop the third", + signers: @["A", "B", "C"], + votes: @[TesterVote(signer: "A", voted: "C"), + TesterVote(signer: "B", voted: "C")], + results: @["A", "B"]), + + TestSpecs( + id: 8, + info: "Four signers, consensus of two not being enough to drop anyone", + signers: @["A", "B", "C", "D"], + votes: @[TesterVote(signer: "A", voted: "C"), + TesterVote(signer: "B", voted: "C")], + results: @["A", "B", "C", "D"]), + + TestSpecs( + id: 9, + info: "Four signers, consensus of three already being enough to " & + "drop someone", + signers: @["A", "B", "C", "D"], + votes: @[TesterVote(signer: "A", voted: "D"), + TesterVote(signer: "B", voted: "D"), + TesterVote(signer: "C", voted: "D")], + results: @["A", "B", "C"]), + + TestSpecs( + id: 10, + info: "Authorizations are counted once per signer per target", + signers: @["A", "B"], + votes: @[TesterVote(signer: "A", voted: "C", auth: true), + TesterVote(signer: "B"), + TesterVote(signer: "A", voted: "C", auth: true), + TesterVote(signer: "B"), + TesterVote(signer: "A", voted: "C", auth: true)], + results: @["A", "B"]), + + TestSpecs( + id: 11, + info: "Authorizing multiple accounts concurrently is permitted", + signers: @["A", "B"], + votes: @[TesterVote(signer: "A", voted: "C", auth: true), + TesterVote(signer: "B"), + TesterVote(signer: "A", voted: "D", auth: true), + TesterVote(signer: "B"), + TesterVote(signer: "A"), + TesterVote(signer: "B", voted: "D", auth: true), + TesterVote(signer: "A"), + TesterVote(signer: "B", voted: "C", auth: true)], + results: @["A", "B", "C", "D"]), + + TestSpecs( + id: 12, + info: "Deauthorizations are counted once per signer per target", + signers: @["A", "B"], + votes: @[TesterVote(signer: "A", voted: "B"), + TesterVote(signer: "B"), + TesterVote(signer: "A", voted: "B"), + TesterVote(signer: "B"), + TesterVote(signer: "A", voted: "B")], + results: @["A", "B"]), + + TestSpecs( + id: 13, + info: "Deauthorizing multiple accounts concurrently is permitted", + signers: @["A", "B", "C", "D"], + votes: @[TesterVote(signer: "A", voted: "C"), + TesterVote(signer: "B"), + TesterVote(signer: "C"), + TesterVote(signer: "A", voted: "D"), + TesterVote(signer: "B"), + TesterVote(signer: "C"), + TesterVote(signer: "A"), + TesterVote(signer: "B", voted: "D"), + TesterVote(signer: "C", voted: "D"), + TesterVote(signer: "A"), + TesterVote(signer: "B", voted: "C")], + results: @["A", "B"]), + + TestSpecs( + id: 14, + info: "Votes from deauthorized signers are discarded immediately " & + "(deauth votes)", + signers: @["A", "B", "C"], + votes: @[TesterVote(signer: "C", voted: "B"), + TesterVote(signer: "A", voted: "C"), + TesterVote(signer: "B", voted: "C"), + TesterVote(signer: "A", voted: "B")], + results: @["A", "B"]), + + TestSpecs( + id: 15, + info: "Votes from deauthorized signers are discarded immediately " & + "(auth votes)", + signers: @["A", "B", "C"], + votes: @[TesterVote(signer: "C", voted: "D", auth: true), + TesterVote(signer: "A", voted: "C"), + TesterVote(signer: "B", voted: "C"), + TesterVote(signer: "A", voted: "D", auth: true)], + results: @["A", "B"]), + + TestSpecs( + id: 16, + info: "Cascading changes are not allowed, only the account being " & + "voted on may change", + signers: @["A", "B", "C", "D"], + votes: @[TesterVote(signer: "A", voted: "C"), + TesterVote(signer: "B"), + TesterVote(signer: "C"), + TesterVote(signer: "A", voted: "D"), + TesterVote(signer: "B", voted: "C"), + TesterVote(signer: "C"), + TesterVote(signer: "A"), + TesterVote(signer: "B", voted: "D"), + TesterVote(signer: "C", voted: "D")], + results: @["A", "B", "C"]), + + TestSpecs( + id: 17, + info: "Changes reaching consensus out of bounds (via a deauth) " & + "execute on touch", + signers: @["A", "B", "C", "D"], + votes: @[TesterVote(signer: "A", voted: "C"), + TesterVote(signer: "B"), + TesterVote(signer: "C"), + TesterVote(signer: "A", voted: "D"), + TesterVote(signer: "B", voted: "C"), + TesterVote(signer: "C"), + TesterVote(signer: "A"), + TesterVote(signer: "B", voted: "D"), + TesterVote(signer: "C", voted: "D"), + TesterVote(signer: "A"), + TesterVote(signer: "C", voted: "C", auth: true)], + results: @["A", "B"]), + + TestSpecs( + id: 18, + info: "Changes reaching consensus out of bounds (via a deauth) " & + "may go out of consensus on first touch", + signers: @["A", "B", "C", "D"], + votes: @[TesterVote(signer: "A", voted: "C"), + TesterVote(signer: "B"), + TesterVote(signer: "C"), + TesterVote(signer: "A", voted: "D"), + TesterVote(signer: "B", voted: "C"), + TesterVote(signer: "C"), + TesterVote(signer: "A"), + TesterVote(signer: "B", voted: "D"), + TesterVote(signer: "C", voted: "D"), + TesterVote(signer: "A"), + TesterVote(signer: "B", voted: "C", auth: true)], + results: @["A", "B", "C"]), + + TestSpecs( + id: 19, + info: "Ensure that pending votes don't survive authorization status " & + "changes. This corner case can only appear if a signer is " & + "quickly added, removed and then readded (or the inverse), " & + "while one of the original voters dropped. If a past vote is " & + "left cached in the system somewhere, this will interfere " & + "with the final signer outcome.", + signers: @["A", "B", "C", "D", "E"], + votes: @[ + # Authorize F, 3 votes needed + TesterVote(signer: "A", voted: "F", auth: true), + TesterVote(signer: "B", voted: "F", auth: true), + TesterVote(signer: "C", voted: "F", auth: true), + + # Deauthorize F, 4 votes needed (leave A's previous vote "unchanged") + TesterVote(signer: "D", voted: "F"), + TesterVote(signer: "E", voted: "F"), + TesterVote(signer: "B", voted: "F"), + TesterVote(signer: "C", voted: "F"), + + # Almost authorize F, 2/3 votes needed + TesterVote(signer: "D", voted: "F", auth: true), + TesterVote(signer: "E", voted: "F", auth: true), + + # Deauthorize A, 3 votes needed + TesterVote(signer: "B", voted: "A"), + TesterVote(signer: "C", voted: "A"), + TesterVote(signer: "D", voted: "A"), + + # Finish authorizing F, 3/3 votes needed + TesterVote(signer: "B", voted: "F", auth: true)], + results: @["B", "C", "D", "E", "F"]), + + TestSpecs( + id: 20, + info: "Epoch transitions reset all votes to allow chain checkpointing", + epoch: 3, + signers: @["A", "B"], + votes: @[TesterVote(signer: "A", voted: "C", auth: true), + TesterVote(signer: "B"), + TesterVote(signer: "A", checkpoint: @["A", "B"]), + TesterVote(signer: "B", voted: "C", auth: true)], + results: @["A", "B"]), + + TestSpecs( + id: 21, + info: "An unauthorized signer should not be able to sign blocks", + signers: @["A"], + votes: @[TesterVote(signer: "B")], + failure: errUnauthorizedSigner), + + TestSpecs( + id: 22, + info: "An authorized signer that signed recenty should not be able " & + "to sign again", + signers: @["A", "B"], + votes: @[TesterVote(signer: "A"), + TesterVote(signer: "A")], + failure: errRecentlySigned), + + TestSpecs( + id: 23, + info: "Recent signatures should not reset on checkpoint blocks " & + "imported in a batch " & + "(https://github.com/ethereum/go-ethereum/issues/17593). "& + "Whilst this seems overly specific and weird, it was a "& + "Rinkeby consensus split.", + epoch: 3, + signers: @["A", "B", "C"], + votes: @[TesterVote(signer: "A"), + TesterVote(signer: "B"), + TesterVote(signer: "A", checkpoint: @["A", "B", "C"]), + TesterVote(signer: "A", newbatch: true)], + failure: errRecentlySigned)] + +static: + # For convenience, make sure that IDs are increasing + for n in 1 ..< voterSamples.len: + if voterSamples[n-1].id < voterSamples[n].id: + continue + echo "voterSamples[", n, "] == ", voterSamples[n].id, " expected ", + voterSamples[n-1].id + 1, " or greater" + doAssert voterSamples[n-1].id < voterSamples[n].id + +# End