Merge pull request #741 from status-im/devel

merge devel into master
This commit is contained in:
Ștefan Talpalaru 2020-02-12 17:58:45 +01:00 committed by GitHub
commit c205e32ed9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
46 changed files with 400 additions and 325 deletions

View File

@ -37,7 +37,7 @@ build_script:
test_script:
# the "go-checks" target fails in AppVeyor, for some reason; easier to disable than to debug
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p"
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_GO_CHECKS=1 P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
- mingw32-make -j2 ARCH_OVERRIDE=%PLATFORM% DISABLE_TEST_FIXTURES_SCRIPT=1 DISABLE_GO_CHECKS=1 test
deploy: off

4
.gitmodules vendored
View File

@ -28,9 +28,9 @@
branch = master
[submodule "vendor/NimYAML"]
path = vendor/NimYAML
url = https://github.com/flyx/NimYAML.git
url = https://github.com/status-im/NimYAML.git
ignore = dirty
branch = master
branch = devel
[submodule "vendor/nim-web3"]
path = vendor/nim-web3
url = https://github.com/status-im/nim-web3.git

View File

@ -50,6 +50,6 @@ script:
# Building Nim-1.0.4 takes up to 10 minutes on Travis - the time limit after which jobs are cancelled for having no output
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update # to allow a newer Nim version to be detected
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}"
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" NIMFLAGS="-d:NETWORK_TYPE=libp2p"
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
- make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" DISABLE_TEST_FIXTURES_SCRIPT=1 test

2
Jenkinsfile vendored
View File

@ -22,7 +22,7 @@ def runStages() {
"tools": {
stage("Tools") {
sh "make -j${env.NPROC}"
sh "make -j${env.NPROC} NIMFLAGS='-d:NETWORK_TYPE=libp2p'"
sh "make -j${env.NPROC} NIMFLAGS='-d:NETWORK_TYPE=libp2p -d:testnet_servers_image'"
}
},
"test suite": {

View File

@ -114,10 +114,10 @@ clean-testnet1:
rm -rf build/data/testnet1
testnet0: | build deps
NIM_PARAMS="$(NIM_PARAMS)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet0
NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet0
testnet1: | build deps
NIM_PARAMS="$(NIM_PARAMS)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet1
NIM_PARAMS="$(NIM_PARAMS)" LOG_LEVEL="$(LOG_LEVEL)" $(ENV_SCRIPT) nim $(NIM_PARAMS) scripts/connect_to_testnet.nims testnet1
clean: | clean-common
rm -rf build/{$(TOOLS_CSV),all_tests,*_node,*ssz*,beacon_node_testnet*,state_sim,transition*}

View File

@ -14,7 +14,7 @@ jobs:
- task: CacheBeta@1
displayName: 'cache Nim binaries'
inputs:
key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)"
key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)" | "v2"
path: NimBinaries
- task: CacheBeta@1
@ -72,7 +72,7 @@ jobs:
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} CI_CACHE=NimBinaries update
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} fetch-dlls
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p"
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} P2PD_CACHE=p2pdCache NIMFLAGS="-d:NETWORK_TYPE=libp2p -d:testnet_servers_image"
file build/beacon_node
mingw32-make -j2 ARCH_OVERRIDE=${PLATFORM} DISABLE_TEST_FIXTURES_SCRIPT=1 test
displayName: 'build and test'

View File

@ -218,6 +218,8 @@ proc init*(T: type BeaconNode, conf: BeaconNodeConf): Future[BeaconNode] {.async
mainchainMonitor = MainchainMonitor.init(
conf.depositWeb3Url, conf.depositContractAddress,
blockPool.headState.data.data.eth1_data.block_hash)
# TODO if we don't have any validators attached, we don't need a mainchain
# monitor
mainchainMonitor.start()
var
@ -624,29 +626,20 @@ proc handleProposal(node: BeaconNode, head: BlockRef, slot: Slot):
# TODO here we advance the state to the new slot, but later we'll be
# proposing for it - basically, we're selecting proposer based on an
# empty slot
var cache = get_empty_per_epoch_cache()
node.blockPool.withState(node.blockPool.tmpState, head.atSlot(slot)):
let proposerIdx = get_beacon_proposer_index(state, cache)
if proposerIdx.isNone:
notice "Missing proposer index",
slot=slot,
epoch=slot.compute_epoch_at_slot,
num_validators=state.validators.len,
active_validators=
get_active_validator_indices(state, slot.compute_epoch_at_slot),
balances=state.balances
let proposerKey = node.blockPool.getProposer(head, slot)
if proposerKey.isNone():
return head
let validator = node.getAttachedValidator(state, proposerIdx.get)
let validator = node.attachedValidators.getValidator(proposerKey.get())
if validator != nil:
return await proposeBlock(node, validator, head, slot)
trace "Expecting block proposal",
debug "Expecting block proposal",
headRoot = shortLog(head.root),
slot = shortLog(slot),
proposer = shortLog(state.validators[proposerIdx.get].pubKey),
proposer = shortLog(proposerKey.get()),
cat = "consensus",
pcs = "wait_for_proposal"
@ -1113,7 +1106,7 @@ when isMainModule:
else: waitFor getLatestEth1BlockHash(config.depositWeb3Url)
var
initialState = initialize_beacon_state_from_eth1(
eth1Hash, startTime, deposits, {skipValidation})
eth1Hash, startTime, deposits, {skipValidation, skipMerkleValidation})
# https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
initialState.genesis_time = startTime

View File

@ -2,7 +2,7 @@ import
bitops, chronicles, options, tables,
ssz, beacon_chain_db, state_transition, extras,
beacon_node_types, metrics,
spec/[crypto, datatypes, digest, helpers]
spec/[crypto, datatypes, digest, helpers, validator]
declareCounter beacon_reorgs_total, "Total occurrences of reorganizations of the chain" # On fork choice
@ -249,9 +249,10 @@ proc init*(T: type BlockPool, db: BeaconChainDB): BlockPool =
res
proc addResolvedBlock(
pool: var BlockPool, state: var StateData, blockRoot: Eth2Digest,
pool: var BlockPool, state: BeaconState, blockRoot: Eth2Digest,
signedBlock: SignedBeaconBlock, parent: BlockRef): BlockRef =
logScope: pcs = "block_resolution"
doAssert state.slot == signedBlock.message.slot, "state must match block"
let blockRef = BlockRef.init(blockRoot, signedBlock.message)
link(parent, blockRef)
@ -262,17 +263,10 @@ proc addResolvedBlock(
# Resolved blocks should be stored in database
pool.db.putBlock(blockRoot, signedBlock)
# TODO this is a bit ugly - we update state.data outside of this function then
# set the rest here - need a blockRef to update it. Clean this up -
# hopefully it won't be necessary by the time hash caching and the rest
# is done..
doAssert state.data.data.slot == blockRef.slot
state.blck = blockRef
# This block *might* have caused a justification - make sure we stow away
# that information:
let justifiedSlot =
state.data.data.current_justified_checkpoint.epoch.compute_start_slot_at_epoch()
state.current_justified_checkpoint.epoch.compute_start_slot_at_epoch()
var foundHead: Option[Head]
for head in pool.heads.mitems():
@ -389,9 +383,12 @@ proc add*(
return
# Careful, pool.tmpState is now partially inconsistent and will be updated
# inside addResolvedBlock
return pool.addResolvedBlock(pool.tmpState, blockRoot, signedBlock, parent)
# Careful, tmpState.data has been updated but not blck - we need to create
# the BlockRef first!
pool.tmpState.blck = pool.addResolvedBlock(
pool.tmpState.data.data, blockRoot, signedBlock, parent)
return pool.tmpState.blck
# TODO already checked hash though? main reason to keep this is because
# the pending pool calls this function back later in a loop, so as long
@ -441,7 +438,6 @@ proc add*(
missing = pool.missing.len,
cat = "filtering"
func getRef*(pool: BlockPool, root: Eth2Digest): BlockRef =
## Retrieve a resolved block reference, if available
pool.blocks.getOrDefault(root, nil)
@ -565,12 +561,19 @@ func checkMissing*(pool: var BlockPool): seq[FetchRecord] =
if v.tries.popcount() == 1:
result.add(FetchRecord(root: k, historySlots: v.slots))
proc skipAndUpdateState(
state: var HashedBeaconState, slot: Slot,
afterUpdate: proc (state: HashedBeaconState)) =
while state.data.slot < slot:
# Process slots one at a time in case afterUpdate needs to see empty states
process_slots(state, state.data.slot + 1)
afterUpdate(state)
proc skipAndUpdateState(
state: var HashedBeaconState, blck: BeaconBlock, flags: UpdateFlags,
afterUpdate: proc (state: HashedBeaconState)): bool =
process_slots(state, blck.slot - 1)
afterUpdate(state)
skipAndUpdateState(state, blck.slot - 1, afterUpdate)
let ok = state_transition(state, blck, flags)
@ -603,7 +606,7 @@ proc rewindState(pool: BlockPool, state: var StateData, bs: BlockSlot):
var ancestors = @[pool.get(bs.blck)]
# Common case: the last block applied is the parent of the block to apply:
if not bs.blck.parent.isNil and state.blck.root == bs.blck.parent.root and
state.data.data.slot < bs.slot:
state.data.data.slot < bs.blck.slot:
return ancestors
# It appears that the parent root of the proposed new block is different from
@ -692,8 +695,8 @@ proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
if state.blck.root == bs.blck.root and state.data.data.slot <= bs.slot:
if state.data.data.slot != bs.slot:
# Might be that we're moving to the same block but later slot
process_slots(state.data, bs.slot)
pool.maybePutState(state.data, bs.blck)
skipAndUpdateState(state.data, bs.slot) do(state: HashedBeaconState):
pool.maybePutState(state, bs.blck)
return # State already at the right spot
@ -714,9 +717,8 @@ proc updateStateData*(pool: BlockPool, state: var StateData, bs: BlockSlot) =
pool.maybePutState(state, ancestors[i].refs)
doAssert ok, "Blocks in database should never fail to apply.."
# TODO check if this triggers rest of state transition, or should
process_slots(state.data, bs.slot)
pool.maybePutState(state.data, bs.blck)
skipAndUpdateState(state.data, bs.slot) do(state: HashedBeaconState):
pool.maybePutState(state, bs.blck)
state.blck = bs.blck
@ -924,3 +926,20 @@ proc preInit*(
db.putTailBlock(blockRoot)
db.putHeadBlock(blockRoot)
db.putStateRoot(blockRoot, state.slot, signedBlock.message.state_root)
proc getProposer*(pool: BlockPool, head: BlockRef, slot: Slot): Option[ValidatorPubKey] =
pool.withState(pool.tmpState, head.atSlot(slot)):
var cache = get_empty_per_epoch_cache()
let proposerIdx = get_beacon_proposer_index(state, cache)
if proposerIdx.isNone:
warn "Missing proposer index",
slot=slot,
epoch=slot.compute_epoch_at_slot,
num_validators=state.validators.len,
active_validators=
get_active_validator_indices(state, slot.compute_epoch_at_slot),
balances=state.balances
return
return some(state.validators[proposerIdx.get()].pubkey)

View File

@ -155,7 +155,7 @@ when networkBackend in [libp2p, libp2pDaemon]:
# that are different from the host address (this is relevant when we
# are running behind a NAT).
var switch = newStandardSwitch(some keys.seckey, hostAddress,
triggerSelf = true, gossip = true)
triggerSelf = true, gossip = false)
result = Eth2Node.init(conf, switch, keys.seckey)
for enr in bootstrapEnrs:
result.addKnownPeer(enr)

View File

@ -25,5 +25,15 @@ type
## TODO need to be careful here, easy to assume that slot number change is
## enough, vs advancing the state - however, making a full state copy
## is expensive also :/
skipMerkleValidation ##\
## When processing deposits, skip verifying the Merkle proof trees of each
## deposit. This is a holdover from both interop issues with the malformed
## proofs and, more currently, nim-beacon-chain's creation of proofs which
## are inconsistent with the current specification. Furthermore several of
## the mocking interfaces deliberately do not create Merkle proofs. Whilst
## this seems less than entirely justifiable, for now enable keeping those
## in place while minimizing the tech debt they create. One, in principle,
## should be able to remove this flag entirely. It is not intrinsically an
## expensive operation to perform.
UpdateFlags* = set[UpdateFlag]

View File

@ -24,13 +24,16 @@ type
QueueElement = (BlockHash, DepositData)
proc init*(T: type MainchainMonitor, web3Url, depositContractAddress: string, startBlock: Eth2Digest): T =
result.new()
result.web3Url = web3Url
result.depositContractAddress = Address.fromHex(depositContractAddress)
result.depositQueue = newAsyncQueue[QueueElement]()
result.eth1Block = BlockHash(startBlock.data)
proc init*(
T: type MainchainMonitor,
web3Url, depositContractAddress: string,
startBlock: Eth2Digest): T =
T(
web3Url: web3Url,
depositContractAddress: Address.fromHex(depositContractAddress),
depositQueue: newAsyncQueue[QueueElement](),
eth1Block: BlockHash(startBlock.data),
)
contract(DepositContract):
proc deposit(pubkey: Bytes48, withdrawalCredentials: Bytes32, signature: Bytes96, deposit_data_root: FixedBytes[32])
@ -113,8 +116,18 @@ proc getGenesis*(m: MainchainMonitor): Future[BeaconState] {.async.} =
return m.genesisState[]
proc getBlockNumber(web3: Web3, hash: BlockHash): Future[Quantity] {.async.} =
debug "Querying block number", hash = $hash
try:
let blk = await web3.provider.eth_getBlockByHash(hash, false)
return blk.number
except CatchableError as exc:
# TODO this doesn't make too much sense really, but what would be a
# reasonable behavior? no idea - the whole algorithm needs to be
# rewritten to match the spec.
notice "Failed to get block number from hash, using current block instead",
hash = $hash, err = exc.msg
return await web3.provider.eth_blockNumber()
proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
if delayBeforeStart != ZeroDuration:
@ -129,8 +142,15 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
error "Web3 server disconnected", ulr = m.web3Url
processFut.cancel()
# TODO this needs to implement follow distance and the rest of the honest
# validator spec..
let startBlkNum = await web3.getBlockNumber(m.eth1Block)
debug "Starting eth1 monitor", fromBlock = startBlkNum.uint64
notice "Monitoring eth1 deposits",
fromBlock = startBlkNum.uint64,
contract = $m.depositContractAddress,
url = m.web3Url
let ns = web3.contractSender(DepositContract, m.depositContractAddress)
@ -139,7 +159,7 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
withdrawalCredentials: Bytes32,
amount: Bytes8,
signature: Bytes96, merkleTreeIndex: Bytes8, j: JsonNode):
try:
let blkHash = BlockHash.fromHex(j["blockHash"].getStr())
let amount = bytes_to_int(array[8, byte](amount))
@ -148,6 +168,8 @@ proc run(m: MainchainMonitor, delayBeforeStart: Duration) {.async.} =
withdrawal_credentials: Eth2Digest(data: array[32, byte](withdrawalCredentials)),
amount: amount,
signature: ValidatorSig.init(array[96, byte](signature)))))
except CatchableError as exc:
warn "Received invalid deposit", err = exc.msg, j
try:
await processFut

View File

@ -53,7 +53,7 @@ proc process_deposit*(
# Process an Eth1 deposit, registering a validator or increasing its balance.
# Verify the Merkle branch
if skipValidation notin flags and not is_valid_merkle_branch(
if skipMerkleValidation notin flags and not is_valid_merkle_branch(
hash_tree_root(deposit.data),
deposit.proof,
DEPOSIT_CONTRACT_TREE_DEPTH + 1,
@ -361,7 +361,8 @@ proc process_registry_updates*(state: var BeaconState) {.nbench.}=
# https://github.com/ethereum/eth2.0-specs/blob/v0.9.4/specs/core/0_beacon-chain.md#is_valid_indexed_attestation
proc is_valid_indexed_attestation*(
state: BeaconState, indexed_attestation: IndexedAttestation): bool =
state: BeaconState, indexed_attestation: IndexedAttestation,
flags: UpdateFlags): bool =
## Check if ``indexed_attestation`` has valid indices and signature.
# TODO: this is noSideEffect besides logging
# https://github.com/status-im/nim-chronicles/issues/62
@ -380,7 +381,7 @@ proc is_valid_indexed_attestation*(
return false
# Verify aggregate signature
if not bls_verify(
if skipValidation notin flags and not bls_verify(
bls_aggregate_pubkeys(mapIt(indices, state.validators[it.int].pubkey)),
hash_tree_root(indexed_attestation.data).data,
indexed_attestation.signature,
@ -497,7 +498,7 @@ proc check_attestation*(
return
if not is_valid_indexed_attestation(
state, get_indexed_attestation(state, attestation, stateCache)):
state, get_indexed_attestation(state, attestation, stateCache), flags):
warn("process_attestation: signature or bitfields incorrect")
return

View File

@ -160,8 +160,10 @@ func bls_verify*(
return false
# TODO bls_verify_multiple(...) used to have this workaround, and now it
# lives here. No matter the signature, there's also no meaningful way to
# verify it -- it's a kind of vacuous truth. No pubkey/sig pairs.
if pubkey == default(ValidatorPubKey):
# verify it -- it's a kind of vacuous truth. No pubkey/sig pairs. Sans a
# getBytes() or similar mechanism, pubKey == default(ValidatorPubKey) is
# a way to create many false positive matches. This seems odd.
if pubkey.getBytes() == default(ValidatorPubKey).getBytes():
return true
sig.blsValue.verify(msg, domain, pubkey.blsValue)

View File

@ -20,12 +20,12 @@
# we call this function `eth2hash`, and it outputs a `Eth2Digest`. Easy to sed :)
import
chronicles,
chronicles, json_serialization,
nimcrypto/[sha2, hash, utils],
hashes
export
hash.`$`
hash.`$`, json_serialization
type
Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
@ -70,3 +70,9 @@ func hash*(x: Eth2Digest): Hash =
# We just slice the first 4 or 8 bytes of the block hash
# depending of if we are on a 32 or 64-bit platform
result = cast[ptr Hash](unsafeAddr x)[]
proc writeValue*(writer: var JsonWriter, value: Eth2Digest) =
writeValue(writer, value.data.toHex(true))
proc readValue*(reader: var JsonReader, value: var Eth2Digest) =
value = Eth2Digest.fromHex(reader.readValue(string))

View File

@ -217,6 +217,7 @@ func is_slashable_attestation_data(
proc process_attester_slashing*(
state: var BeaconState,
attester_slashing: AttesterSlashing,
flags: UpdateFlags,
stateCache: var StateCache
): bool {.nbench.}=
let
@ -228,11 +229,11 @@ proc process_attester_slashing*(
notice "Attester slashing: surround or double vote check failed"
return false
if not is_valid_indexed_attestation(state, attestation_1):
if not is_valid_indexed_attestation(state, attestation_1, flags):
notice "Attester slashing: invalid attestation 1"
return false
if not is_valid_indexed_attestation(state, attestation_2):
if not is_valid_indexed_attestation(state, attestation_2, flags):
notice "Attester slashing: invalid attestation 2"
return false
@ -259,7 +260,7 @@ proc processAttesterSlashings(state: var BeaconState, blck: BeaconBlock,
return false
for attester_slashing in blck.body.attester_slashings:
if not process_attester_slashing(state, attester_slashing, stateCache):
if not process_attester_slashing(state, attester_slashing, {}, stateCache):
return false
return true
@ -286,13 +287,14 @@ proc processAttestations(
true
# https://github.com/ethereum/eth2.0-specs/blob/v0.8.4/specs/core/0_beacon-chain.md#deposits
proc processDeposits(state: var BeaconState, blck: BeaconBlock): bool {.nbench.}=
proc processDeposits(state: var BeaconState, blck: BeaconBlock,
flags: UpdateFlags): bool {.nbench.}=
if not (len(blck.body.deposits) <= MAX_DEPOSITS):
notice "processDeposits: too many deposits"
return false
for deposit in blck.body.deposits:
if not process_deposit(state, deposit):
if not process_deposit(state, deposit, flags):
notice "processDeposits: deposit invalid"
return false
@ -419,7 +421,7 @@ proc processBlock*(
debug "[Block processing] Attestation processing failure", slot = shortLog(state.slot)
return false
if not processDeposits(state, blck):
if not processDeposits(state, blck, flags):
debug "[Block processing] Deposit processing failure", slot = shortLog(state.slot)
return false

24
beacon_chain/sszdump.nim Normal file
View File

@ -0,0 +1,24 @@
import
os,
ssz,
serialization,
beacon_node_types,
./spec/[crypto, datatypes, digest]
proc dump*(dir: string, v: AttestationData, validator: ValidatorPubKey) =
SSZ.saveFile(
dir / "att-" & $v.slot & "-" &
$v.index & "-" & validator.shortLog &
".ssz", v)
proc dump*(dir: string, v: SignedBeaconBlock, blck: BlockRef) =
SSZ.saveFile(
dir / "block-" & $v.message.slot & "-" &
shortLog(blck.root) & ".ssz", v)
proc dump*(dir: string, v: HashedBeaconState, blck: BlockRef) =
SSZ.saveFile(
dir / "state-" & $v.data.slot & "-" &
shortLog(blck.root) & "-" & shortLog(v.root) & ".ssz",
v.data)

View File

@ -83,7 +83,11 @@ func get_epoch_validator_count(state: BeaconState): int64 =
# https://github.com/ethereum/eth2.0-specs/blob/v0.10.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function
proc process_slots*(state: var BeaconState, slot: Slot) {.nbench.}=
doAssert state.slot <= slot
if not (state.slot <= slot):
warn("Trying to apply old block",
state_slot = state.slot,
slot = slot)
return
# Catch up to the target slot
while state.slot < slot:
@ -197,6 +201,7 @@ proc process_slots*(state: var HashedBeaconState, slot: Slot) =
if state.data.slot > slot:
notice(
"Unusual request for a slot in the past",
state_root = shortLog(state.root),
current_slot = state.data.slot,
target_slot = slot
)

View File

@ -21,12 +21,13 @@ RUN cd /root \
# to get a fresh up-to-date version of Nim and p2pd.
ARG GIT_REVISION
ARG NETWORK_NIM_FLAGS
ARG MARCH_NIM_FLAGS
RUN cd /root/nim-beacon-chain \
&& git fetch \
&& git reset --hard ${GIT_REVISION} \
&& make -j$(nproc) update \
&& make LOG_LEVEL=DEBUG NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
&& make LOG_LEVEL=TRACE NIMFLAGS="-d:debug -d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS} ${MARCH_NIM_FLAGS}" beacon_node
# --------------------------------- #
# Starting new image to reduce size #

View File

@ -5,6 +5,8 @@ SHELL := bash # the shell used internally by "make"
GIT_REVISION ?= $(shell git rev-parse HEAD)
NETWORK ?= testnet1
NETWORK_NIM_FLAGS ?= $(shell ../scripts/load-testnet-nim-flags.sh $(NETWORK))
# Get the required GCC flags by running `gcc -v -E - -march=native </dev/null 2>&1 | grep cc1` on the server.
MARCH_NIM_FLAGS ?= -d:disableMarchNative --passC:'-march=znver1 --param l1-cache-size=32 --param l1-cache-line-size=64 --param l2-cache-size=512'
IMAGE_TAG ?= $(NETWORK)
IMAGE_NAME ?= statusteam/nimbus_beacon_node:$(IMAGE_TAG)
@ -19,6 +21,7 @@ build:
docker build \
--build-arg="GIT_REVISION=$(GIT_REVISION)" \
--build-arg="NETWORK_NIM_FLAGS=$(NETWORK_NIM_FLAGS)" \
--build-arg="MARCH_NIM_FLAGS=$(MARCH_NIM_FLAGS)" \
-t $(IMAGE_NAME) \
--progress=plain \
.

View File

@ -98,7 +98,12 @@ iterator validatorAssignments: tuple[node: Node; firstValidator, lastValidator:
case conf.cmd
of restart_nodes:
for n in nodes():
echo &"ssh {n.server} docker restart {n.container}"
if n.id mod 2 == 0:
# This will only print one line: "docker.io/statusteam/nimbus_beacon_node:testnet1".
echo &"ssh {n.server} docker pull -q statusteam/nimbus_beacon_node:{conf.network}"
# docker-compose will rebuild the container if it detects a newer image.
# Prints: "Recreating beacon-node-testnet1-1 ... done".
echo &"ssh {n.server} 'cd /docker/{n.container} && docker-compose up -d'"
of reset_network:
for n, firstValidator, lastValidator in validatorAssignments():
@ -114,8 +119,9 @@ of reset_network:
let dockerPath = &"/docker/{n.container}/data/BeaconNode"
echo &"echo Syncing {lastValidator - firstValidator} keys starting from {firstValidator} to container {n.container}@{n.server} ... && \\"
echo &" ssh {n.server} 'sudo rm -rf /tmp/nimbus && mkdir -p /tmp/nimbus/' && \\"
echo &" rsync {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
if keysList.len > 0: echo &" rsync {keysList} {n.server}:/tmp/nimbus/keys/ && \\"
echo &" rsync -a -zz {networkDataFiles} {n.server}:/tmp/nimbus/net-data/ && \\"
if keysList.len > 0:
echo &" rsync -a -zz {keysList} {n.server}:/tmp/nimbus/keys/ && \\"
echo &" ssh {n.server} 'sudo docker container stop {n.container} && " &
&"sudo mkdir -p {dockerPath}/validators && " &

View File

@ -32,3 +32,4 @@ cli do(kind: string, file: string):
of "state": printit(BeaconState)
of "proposer_slashing": printit(ProposerSlashing)
of "voluntary_exit": printit(VoluntaryExit)
else: echo "Unknown kind"

16
nim.cfg
View File

@ -21,6 +21,22 @@
-d:"chronicles_colors=off"
@end
# This helps especially for 32-bit x86, which sans SSE2 and newer instructions
# requires quite roundabout code generation for cryptography, and other 64-bit
# and larger arithmetic use cases, along with register starvation issues. When
# engineering a more portable binary release, this should be tweaked but still
# use at least -msse2 or -msse3.
@if disableMarchNative:
--passC:"-msse3"
@else:
--passC:"-march=native"
@if windows:
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782
# ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes)
--passC:"-mno-avx512vl"
@end
@end
--threads:on
--opt:speed
--excessiveStackTrace:on

View File

@ -79,7 +79,7 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
let
genesisState =
initialize_beacon_state_from_eth1(
Eth2Digest(), 0, deposits, {skipValidation})
Eth2Digest(), 0, deposits, {skipMerkleValidation})
genesisBlock = get_initial_beacon_block(genesisState)
echo "Starting simulation..."

View File

@ -9,6 +9,7 @@ const
genesisFile = "genesis.ssz"
configFile = "config.yaml"
testnetsRepo = "eth2-testnets"
web3Url = "wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a"
let
testnetsOrg = getEnv("ETH2_TESTNETS_ORG", "eth2-clients")
@ -75,6 +76,7 @@ cli do (testnetName {.argument.}: string):
validatorsDir = dataDir / "validators"
dumpDir = dataDir / "dump"
beaconNodeBinary = buildDir / "beacon_node_" & dataDirName
var
nimFlags = "-d:chronicles_log_level=TRACE " & getEnv("NIM_PARAMS")
let depositContractFile = testnetDir / depositContractFileName
@ -95,6 +97,8 @@ cli do (testnetName {.argument.}: string):
rmDir dataDir
cd rootDir
if testnet == "testnet1":
nimFlags &= " -d:NETWORK_TYPE=libp2p"
exec &"""nim c {nimFlags} -d:"const_preset={preset}" -o:"{beaconNodeBinary}" beacon_chain/beacon_node.nim"""
mkDir dumpDir
@ -120,15 +124,24 @@ cli do (testnetName {.argument.}: string):
--random-deposits=1
--deposits-dir="{validatorsDir}"
--deposit-private-key={privKey}
--web3-url=wss://goerli.infura.io/ws/v3/809a18497dd74102b5f37d25aae3c85a
--web3-url={web3Url}
{depositContractOpt}
""", "\n", " ")
quit()
mode = Silent
echo "\nDeposit sent, wait for confirmation then press enter to continue"
discard readLineFromStdin()
let logLevel = getEnv("LOG_LEVEL")
var logLevelOpt = ""
if logLevel.len > 0:
logLevelOpt = "--log-level=" & logLevel
mode = Verbose
execIgnoringExitCode replace(&"""{beaconNodeBinary}
--data-dir="{dataDir}"
--dump=true
--web3-url={web3Url}
{bootstrapFileOpt}
{logLevelOpt}
--state-snapshot="{testnetDir/genesisFile}" """ & depositContractOpt, "\n", " ")

View File

@ -32,11 +32,10 @@ echo "Beacon node data dir : ${DATA_DIR:="build/testnet-reset-data/$NETWORK"}
echo "Nim build flags : $NETWORK_NIM_FLAGS"
while true; do
read -p "Continue? [yn] " yn
read -p "Continue? [Yn] " yn
case $yn in
[Yy]* ) break;;
* ) break;;
[Nn]* ) exit 1;;
* ) echo "Please answer yes or no.";;
esac
done
@ -56,41 +55,42 @@ fi
mkdir -p "$DEPOSITS_DIR_ABS"
DOCKER_BEACON_NODE="docker run -v $DEPOSITS_DIR_ABS:/deposits_dir -v $NETWORK_DIR_ABS:/network_dir -v $DATA_DIR_ABS:/data_dir statusteam/nimbus_beacon_node:$NETWORK"
make deposit_contract
if [ "$ETH1_PRIVATE_KEY" != "" ]; then
make deposit_contract
echo "Deploying deposit contract through $WEB3_URL_ARG..."
DEPOSIT_CONTRACT_ADDRESS=$(./build/deposit_contract deploy $WEB3_URL_ARG --private-key=$ETH1_PRIVATE_KEY)
DEPOSIT_CONTRACT_ADDRESS_ARG="--deposit-contract=$DEPOSIT_CONTRACT_ADDRESS"
echo "Done: $DEPOSIT_CONTRACT_ADDRESS"
fi
echo "Building a local beacon_node instance for 'makeDeposits' and 'createTestnet'"
make NIMFLAGS="-d:insecure -d:testnet_servers_image ${NETWORK_NIM_FLAGS}" beacon_node
cd docker
echo "Building Docker image..."
# CPU-specific CFLAGS that work on the servers are in MARCH_NIM_FLAGS,
# in docker/Makefile, and are enabled by default.
make build
$DOCKER_BEACON_NODE makeDeposits \
../build/beacon_node makeDeposits \
--quickstart-deposits=$QUICKSTART_VALIDATORS \
--random-deposits=$RANDOM_VALIDATORS \
--deposits-dir=/deposits_dir
--deposits-dir="$DEPOSITS_DIR_ABS"
TOTAL_VALIDATORS="$(( $QUICKSTART_VALIDATORS + $RANDOM_VALIDATORS ))"
$DOCKER_BEACON_NODE \
--data-dir=/data_dir \
createTestnet \
--validators-dir=/deposits_dir \
../build/beacon_node createTestnet \
--data-dir="$DATA_DIR_ABS" \
--validators-dir="$DEPOSITS_DIR_ABS" \
--total-validators=$TOTAL_VALIDATORS \
--last-user-validator=$QUICKSTART_VALIDATORS \
--output-genesis=/network_dir/genesis.ssz \
--output-bootstrap-file=/network_dir/bootstrap_nodes.txt \
--output-genesis="$NETWORK_DIR_ABS/genesis.ssz" \
--output-bootstrap-file="$NETWORK_DIR_ABS/bootstrap_nodes.txt" \
--bootstrap-address=$BOOTSTRAP_IP \
--bootstrap-port=$BOOTSTRAP_PORT \
$WEB3_URL_ARG $DEPOSIT_CONTRACT_ADDRESS_ARG \
--genesis-offset=900 # Delay in seconds
--genesis-offset=300 # Delay in seconds
COMMITTED_FILES=" genesis.ssz bootstrap_nodes.txt "

View File

@ -25,7 +25,7 @@ proc initGenesisState*(num_validators: uint64, genesis_time: uint64 = 0): Beacon
)
initialize_beacon_state_from_eth1(
eth1BlockHash, 0, deposits, {skipValidation})
eth1BlockHash, 0, deposits, {skipValidation, skipMerkleValidation})
when isMainModule:
# Smoke test

View File

@ -20,13 +20,13 @@ import
const OperationsAttestationsDir = SszTestsDir/const_preset/"phase0"/"operations"/"attestation"/"pyspec_tests"
template runTest(testName: string, identifier: untyped) =
proc runTest(identifier: string) =
# We wrap the tests in a proc to avoid running out of globals
# in the future: Nim supports up to 3500 globals
# but unittest with the macro/templates put everything as globals
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
const testDir = OperationsAttestationsDir / astToStr(identifier)
let testDir = OperationsAttestationsDir / identifier
proc `testImpl _ operations_attestations _ identifier`() =
@ -39,7 +39,7 @@ template runTest(testName: string, identifier: untyped) =
else:
prefix = "[Invalid] "
timedTest prefix & testName & " (" & astToStr(identifier) & ")":
timedTest prefix & identifier:
var stateRef, postRef: ref BeaconState
var attestationRef: ref Attestation
new attestationRef
@ -66,27 +66,5 @@ template runTest(testName: string, identifier: untyped) =
`testImpl _ operations_attestations _ identifier`()
suite "Official - Operations - Attestations " & preset():
# https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/attestation/pyspec_tests
# https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/attestation/pyspec_tests
runTest("after_epoch_slots", after_epoch_slots)
runTest("bad source root", bad_source_root)
runTest("before inclusion delay", before_inclusion_delay)
runTest("empty aggregation bits", empty_aggregation_bits)
runTest("future target epoch", future_target_epoch)
runTest("invalid attestation signature", invalid_attestation_signature)
runTest("invalid current source root", invalid_current_source_root)
runTest("invalid index", invalid_index)
runTest("mismatched target and slot", mismatched_target_and_slot)
runTest("new source epoch", new_source_epoch)
runTest("old source epoch", old_source_epoch)
runTest("old target epoch", old_target_epoch)
runTest("source root is target root", source_root_is_target_root)
runTest("success", success)
runTest("success multi-proposer index interations",
success_multi_proposer_index_iterations)
runTest("success previous epoch", success_previous_epoch)
runTest("too few aggregation bits", too_few_aggregation_bits)
runTest("too many aggregation bits", too_many_aggregation_bits)
runTest("wrong index for committee signature",
wrong_index_for_committee_signature)
runTest("wrong index for slot", wrong_index_for_slot)
for kind, path in walkDir(OperationsAttestationsDir, true):
runTest(path)

View File

@ -12,7 +12,7 @@ import
os, unittest,
# Beacon chain internals
../../beacon_chain/spec/[datatypes, state_transition_block, validator],
../../beacon_chain/ssz,
../../beacon_chain/[extras, ssz],
# Test utilities
../testutil,
./fixtures_utils,
@ -20,23 +20,26 @@ import
const OpAttSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"attester_slashing"/"pyspec_tests"
template runTest(identifier: untyped) =
proc runTest(identifier: string) =
# We wrap the tests in a proc to avoid running out of globals
# in the future: Nim supports up to 3500 globals
# but unittest with the macro/templates put everything as globals
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
const testDir = OpAttSlashingDir / astToStr(identifier)
let testDir = OpAttSlashingDir / identifier
proc `testImpl _ operations_attester_slashing _ identifier`() =
var flags: UpdateFlags
var prefix: string
if not existsFile(testDir/"meta.yaml"):
flags.incl skipValidation
if existsFile(testDir/"post.ssz"):
prefix = "[Valid] "
else:
prefix = "[Invalid] "
timedTest prefix & astToStr(identifier):
timedTest prefix & identifier:
var stateRef, postRef: ref BeaconState
var attesterSlashingRef: ref AttesterSlashing
new attesterSlashingRef
@ -52,10 +55,12 @@ template runTest(identifier: untyped) =
postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
if postRef.isNil:
let done = process_attester_slashing(stateRef[], attesterSlashingRef[], cache)
let done = process_attester_slashing(stateRef[], attesterSlashingRef[],
flags, cache)
doAssert done == false, "We didn't expect this invalid attester slashing to be processed."
else:
let done = process_attester_slashing(stateRef[], attesterSlashingRef[], cache)
let done = process_attester_slashing(stateRef[], attesterSlashingRef[],
flags, cache)
doAssert done, "Valid attestater slashing not processed"
check: stateRef.hash_tree_root() == postRef.hash_tree_root()
reportDiff(stateRef, postRef)
@ -63,19 +68,17 @@ template runTest(identifier: untyped) =
`testImpl _ operations_attester_slashing _ identifier`()
suite "Official - Operations - Attester slashing " & preset():
runTest(success_double)
runTest(success_surround)
runTest(success_already_exited_recent)
runTest(success_already_exited_long_ago)
runTest(invalid_sig_1)
when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
runTest(invalid_sig_2)
runTest(invalid_sig_1_and_2)
runTest(same_data)
runTest(no_double_or_surround)
runTest(participants_already_slashed)
when false: # TODO - https://github.com/status-im/nim-beacon-chain/issues/429
runTest(att1_bad_extra_index)
runTest(att1_bad_replaced_index)
runTest(att2_bad_extra_index)
runTest(att2_bad_replaced_index)
# TODO these are both valid and check BLS signatures, which isn't working
# since 0.10.x introduces new BLS signing/verifying interface with domain
# in particular handled differently through compute_signing_root() rather
# than through the bls_verify(...) call directly. This did not become the
# visible issue it now is because another bug had been masking it wherein
# crypto.nim's bls_verify(...) call had been creating false positives, in
# which cases signature checks had been incorrectly passing.
const expected_failures =
["success_already_exited_recent", "success_already_exited_long_ago"]
for kind, path in walkDir(OpAttSlashingDir, true):
if path in expected_failures:
echo "Skipping test: ", path
continue
runTest(path)

View File

@ -20,26 +20,23 @@ import
const OpBlockHeaderDir = SszTestsDir/const_preset/"phase0"/"operations"/"block_header"/"pyspec_tests"
template runTest(identifier: untyped) =
proc runTest(identifier: string) =
# We wrap the tests in a proc to avoid running out of globals
# in the future: Nim supports up to 3500 globals
# but unittest with the macro/templates put everything as globals
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
const testDir = OpBlockHeaderDir / astToStr(identifier)
let testDir = OpBlockHeaderDir / identifier
proc `testImpl _ blockheader _ identifier`() =
var flags: UpdateFlags
var prefix: string
if not existsFile(testDir/"meta.yaml"):
flags.incl skipValidation
if existsFile(testDir/"post.ssz"):
prefix = "[Valid] "
else:
prefix = "[Invalid] "
timedTest prefix & astToStr(identifier):
timedTest prefix & identifier:
var stateRef, postRef: ref BeaconState
var blck: ref BeaconBlock
new blck
@ -55,10 +52,10 @@ template runTest(identifier: untyped) =
postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
if postRef.isNil:
let done = process_block_header(stateRef[], blck[], flags, cache)
let done = process_block_header(stateRef[], blck[], {}, cache)
doAssert done == false, "We didn't expect this invalid block header to be processed."
else:
let done = process_block_header(stateRef[], blck[], flags, cache)
let done = process_block_header(stateRef[], blck[], {}, cache)
doAssert done, "Valid block header not processed"
check: stateRef.hash_tree_root() == postRef.hash_tree_root()
reportDiff(stateRef, postRef)
@ -66,8 +63,5 @@ template runTest(identifier: untyped) =
`testImpl _ blockheader _ identifier`()
suite "Official - Operations - Block header " & preset():
runTest(success_block_header)
runTest(invalid_slot_block_header)
when false: # skipValidation needs to be split https://github.com/status-im/nim-beacon-chain/issues/407
runTest(invalid_parent_root)
runTest(proposer_slashed)
for kind, path in walkDir(OpBlockHeaderDir, true):
runTest(path)

View File

@ -20,13 +20,13 @@ import
const OperationsDepositsDir = SszTestsDir/const_preset/"phase0"/"operations"/"deposit"/"pyspec_tests"
template runTest(testName: string, identifier: untyped) =
proc runTest(identifier: string) =
# We wrap the tests in a proc to avoid running out of globals
# in the future: Nim supports up to 3500 globals
# but unittest with the macro/templates put everything as globals
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
const testDir = OperationsDepositsDir / astToStr(identifier)
let testDir = OperationsDepositsDir / identifier
proc `testImpl _ operations_deposits _ identifier`() =
@ -39,7 +39,7 @@ template runTest(testName: string, identifier: untyped) =
else:
prefix = "[Invalid] "
timedTest prefix & testName & " (" & astToStr(identifier) & ")":
timedTest prefix & " " & identifier:
var stateRef, postRef: ref BeaconState
var depositRef: ref Deposit
new depositRef
@ -53,8 +53,7 @@ template runTest(testName: string, identifier: untyped) =
postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
if postRef.isNil:
expect(AssertionError):
discard process_deposit(stateRef[], depositRef[], flags)
check not process_deposit(stateRef[], depositRef[], flags)
else:
discard process_deposit(stateRef[], depositRef[], flags)
reportDiff(stateRef, postRef)
@ -62,17 +61,11 @@ template runTest(testName: string, identifier: untyped) =
`testImpl _ operations_deposits _ identifier`()
suite "Official - Operations - Deposits " & preset():
runTest("new deposit under max", new_deposit_under_max)
runTest("new deposit max", new_deposit_max)
runTest("new deposit over max", new_deposit_over_max)
runTest("invalid signature new deposit", invalid_sig_new_deposit)
runTest("success top-up", success_top_up)
runTest("invalid signature top-up", invalid_sig_top_up)
runTest("invalid withdrawal credentials top-up", invalid_withdrawal_credentials_top_up)
# TODO
const expected_failures = ["valid_sig_but_forked_state"]
when false:
# TODO - those should give an exception but do not
# probably because skipValidation is too strong
# https://github.com/status-im/nim-beacon-chain/issues/407
runTest("wrong deposit for deposit count", wrong_deposit_for_deposit_count)
runTest("bad merkle proof", bad_merkle_proof)
for kind, path in walkDir(OperationsDepositsDir, true):
if path in expected_failures:
echo "Skipping test: ", path
continue
runTest(path)

View File

@ -20,13 +20,13 @@ import
const OpProposerSlashingDir = SszTestsDir/const_preset/"phase0"/"operations"/"proposer_slashing"/"pyspec_tests"
template runTest(identifier: untyped) =
proc runTest(identifier: string) =
# We wrap the tests in a proc to avoid running out of globals
# in the future: Nim supports up to 3500 globals
# but unittest with the macro/templates put everything as globals
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
const testDir = OpProposerSlashingDir / astToStr(identifier)
let testDir = OpProposerSlashingDir / identifier
proc `testImpl_proposer_slashing _ identifier`() =
@ -66,15 +66,5 @@ template runTest(identifier: untyped) =
`testImpl_proposer_slashing _ identifier`()
suite "Official - Operations - Proposer slashing " & preset():
# https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/proposer_slashing/pyspec_tests
# https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/proposer_slashing/pyspec_tests
runTest(epochs_are_different)
runTest(headers_are_same)
runTest(invalid_proposer_index)
runTest(invalid_sig_1)
runTest(invalid_sig_1_and_2)
runTest(invalid_sig_2)
runTest(proposer_is_not_activated)
runTest(proposer_is_slashed)
runTest(proposer_is_withdrawn)
runTest(success)
for kind, path in walkDir(OpProposerSlashingDir, true):
runTest(path)

View File

@ -20,13 +20,13 @@ import
const OpVoluntaryExitDir = SszTestsDir/const_preset/"phase0"/"operations"/"voluntary_exit"/"pyspec_tests"
template runTest(identifier: untyped) =
proc runTest(identifier: string) =
# We wrap the tests in a proc to avoid running out of globals
# in the future: Nim supports up to 3500 globals
# but unittest with the macro/templates put everything as globals
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
const testDir = OpVoluntaryExitDir / astToStr(identifier)
let testDir = OpVoluntaryExitDir / identifier
proc `testImpl _ voluntary_exit _ identifier`() =
@ -39,7 +39,7 @@ template runTest(identifier: untyped) =
else:
prefix = "[Invalid] "
timedTest prefix & astToStr(identifier):
timedTest prefix & identifier:
var stateRef, postRef: ref BeaconState
var voluntaryExit: ref SignedVoluntaryExit
new voluntaryExit
@ -64,18 +64,5 @@ template runTest(identifier: untyped) =
`testImpl _ voluntary_exit _ identifier`()
suite "Official - Operations - Voluntary exit " & preset():
# https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/operations/voluntary_exit/pyspec_tests
# https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/operations/voluntary_exit/pyspec_tests
runTest(success)
when false:
# TODO not sure how this particularly could falsely succeed
runTest(invalid_signature)
runTest(validator_invalid_validator_index)
runTest(validator_already_exited)
runTest(success_exit_queue)
runTest(validator_exit_in_future)
runTest(default_exit_epoch_subsequent_exit)
runTest(validator_not_active_long_enough)
runTest(validator_not_active)
for kind, path in walkDir(OpVoluntaryExitDir, true):
runTest(path)

View File

@ -9,7 +9,7 @@
import
# Standard library
os, unittest,
os, sequtils, unittest,
# Beacon chain internals
../../beacon_chain/spec/[crypto, datatypes],
../../beacon_chain/[ssz, state_transition, extras],
@ -20,90 +20,55 @@ import
const SanityBlocksDir = SszTestsDir/const_preset/"phase0"/"sanity"/"blocks"/"pyspec_tests"
template runValidTest(testName: string, identifier: untyped, num_blocks: int): untyped =
proc runTest(identifier: string) =
# We wrap the tests in a proc to avoid running out of globals
# in the future: Nim supports up to 3500 globals
# but unittest with the macro/templates put everything as globals
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
const testDir = SanityBlocksDir / astToStr(identifier)
let testDir = SanityBlocksDir / identifier
proc `testImpl _ blck _ identifier`() =
timedTest "[Valid] " & testName & " (" & astToStr(identifier) & ")":
let prefix = if existsFile(testDir/"post.ssz"):
"[Valid] "
else:
"[Invalid] "
timedTest prefix & identifier:
var stateRef, postRef: ref BeaconState
new stateRef
new postRef
stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
if existsFile(testDir/"post.ssz"):
new postRef
postRef[] = parseTest(testDir/"post.ssz", SSZ, BeaconState)
for i in 0 ..< num_blocks:
# In test cases with more than 10 blocks the first 10 aren't 0-prefixed,
# so purely lexicographic sorting wouldn't sort properly.
for i in 0 ..< toSeq(walkPattern(testDir/"blocks_*.ssz")).len:
let blck = parseTest(testDir/"blocks_" & $i & ".ssz", SSZ, SignedBeaconBlock)
if postRef.isNil:
let success = state_transition(stateRef[], blck.message, flags = {})
doAssert not success, "We didn't expect this invalid block to be processed"
else:
# TODO: The EF is using invalid BLS keys so we can't verify them
let success = state_transition(stateRef[], blck.message, flags = {skipValidation})
doAssert success, "Failure when applying block " & $i
# Checks:
# check: stateRef.hash_tree_root() == postRef.hash_tree_root()
if not postRef.isNil:
reportDiff(stateRef, postRef)
`testImpl _ blck _ identifier`()
suite "Official - Sanity - Blocks " & preset():
timedTest "[Invalid] Previous slot block transition (prev_slot_block_transition)":
const testDir = SanityBlocksDir/"prev_slot_block_transition"
var stateRef: ref BeaconState
new stateRef
stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
# Failing due to signature checking in indexed validation checking pending
# 0.10 BLS verification API with new domain handling.
const expected_failures = ["attester_slashing"]
let blck = parseTest(testDir/"blocks_0.ssz", SSZ, SignedBeaconBlock)
# Check that a block build for an old slot cannot be used for state transition
expect(AssertionError):
# assert in process_slots. This should not be triggered
# for blocks from block_pool/network
discard state_transition(stateRef[], blck.message, flags = {skipValidation})
runValidTest("Same slot block transition", same_slot_block_transition, 1)
runValidTest("Empty block transition", empty_block_transition, 1)
when false: # TODO: we need more granular skipValidation
timedTest "[Invalid] Invalid state root":
const testDir = SanityBlocksDir/"invalid_state_root"
var stateRef: ref BeaconState
new stateRef
stateRef[] = parseTest(testDir/"pre.ssz", SSZ, BeaconState)
let blck = parseTest(testDir/"blocks_0.ssz", SSZ, BeaconBlock)
expect(AssertionError):
discard state_transition(stateRef[], blck, flags = {skipValidation})
runValidTest("Skipped Slots", skipped_slots, 1)
runValidTest("Empty epoch transition", empty_epoch_transition, 1)
when const_preset=="minimal":
runValidTest("Empty epoch transition not finalizing", empty_epoch_transition_not_finalizing, 1)
when false:
# TODO investigate/fix after 0.9.0 transition broke this in mainnet and
# in 0.9.1 even minimal broke. For the latter at least, it differs only
# in latest_block_header.body_root, which is just a hash_tree_root() of
# the one block read by this test case. All balances agree. It's an SSZ
# or hashing issue.
runValidTest("Attester slashing", attester_slashing, 1)
runValidTest("Proposer slashing", proposer_slashing, 1)
# TODO: Expected deposit in block
runValidTest("Deposit in block", deposit_in_block, 1)
runValidTest("Deposit top up", deposit_top_up, 1)
when const_preset=="minimal":
# TODO this doesn't work on mainnet
runValidTest("Attestation", attestation, 2)
runValidTest("Voluntary exit", voluntary_exit, 2)
runValidTest("Balance-driven status transitions", balance_driven_status_transitions, 1)
runValidTest("Historical batch", historical_batch, 1)
when const_preset=="minimal":
runValidTest("ETH1 data votes consensus", eth1_data_votes_consensus, 17)
runValidTest("ETH1 data votes no consensus", eth1_data_votes_no_consensus, 16)
for kind, path in walkDir(SanityBlocksDir, true):
if path in expected_failures:
echo "Skipping test: ", path
continue
runTest(path)

View File

@ -9,7 +9,7 @@
import
# Standard library
os, unittest,
os, strutils, unittest,
# Beacon chain internals
../../beacon_chain/spec/datatypes,
../../beacon_chain/state_transition,
@ -20,16 +20,17 @@ import
const SanitySlotsDir = SszTestsDir/const_preset/"phase0"/"sanity"/"slots"/"pyspec_tests"
template runTest(testName: string, identifier: untyped, num_slots: uint64): untyped =
proc runTest(identifier: string) =
# We wrap the tests in a proc to avoid running out of globals
# in the future: Nim supports up to 3500 globals
# but unittest with the macro/templates put everything as globals
# https://github.com/nim-lang/Nim/issues/12084#issue-486866402
const testDir = SanitySlotsDir / astToStr(identifier)
let
testDir = SanitySlotsDir / identifier
num_slots = readLines(testDir / "slots.yaml", 2)[0].parseInt.uint64
proc `testImpl _ slots _ identifier`() =
timedTest "Slots - " & testName & " (" & astToStr(identifier) & ")":
timedTest "Slots - " & identifier:
var stateRef, postRef: ref BeaconState
new stateRef
new postRef
@ -43,18 +44,6 @@ template runTest(testName: string, identifier: untyped, num_slots: uint64): unty
`testImpl _ slots _ identifier`()
# 1 slot
# ---------------------------------------------------------------
suite "Official - Sanity - Slots " & preset():
# https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/minimal/phase0/sanity/slots/pyspec_tests
# https://github.com/ethereum/eth2.0-spec-tests/tree/v0.10.1/tests/mainnet/phase0/sanity/slots/pyspec_tests
runTest("Advance 1 slot", slots_1, 1)
runTest("Advance 2 slots", slots_2, 2)
runTest("Advance an empty epoch", empty_epoch, SLOTS_PER_EPOCH)
const DoubleEpoch = SLOTS_PER_EPOCH.uint64*2 # workaround undeclared identifier "double_empty_epoch"
runTest("Advance 2 empty epochs", double_empty_epoch, DoubleEpoch)
# This starts in the middle of an epoch
runTest("Advance over an epoch boundary", over_epoch_boundary, SLOTS_PER_EPOCH)
for kind, path in walkDir(SanitySlotsDir, true):
runTest(path)

View File

@ -52,7 +52,7 @@ if [[ $NODE_ID == $MASTER_NODE ]]; then
NODE_BIN=$BOOTSTRAP_NODE_BIN
fi
$NODE_BIN \
cd "$DATA_DIR" && $NODE_BIN \
--bootstrap-file=$BOOTSTRAP_ADDRESS_FILE \
--data-dir=$DATA_DIR \
--node-name=$NODE_ID \

View File

@ -16,13 +16,13 @@ mkdir -p "$VALIDATORS_DIR"
cd "$GIT_ROOT"
NIMFLAGS="-d:chronicles_log_level=TRACE --hints:off --warnings:off --verbosity:0 --opt:speed --debuginfo"
NIMFLAGS="-d:chronicles_log_level=TRACE -d:chronicles_sinks:textlines,json[file] --hints:off --warnings:off --verbosity:0 --opt:speed --debuginfo"
# Run with "SLOTS_PER_EPOCH=8 ./start.sh" to change these
DEFS=""
DEFS+="-d:MAX_COMMITTEES_PER_SLOT=${MAX_COMMITTEES_PER_SLOT:-1} " # Spec default: 64
DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-16} " # Spec default: 32
DEFS+="-d:SLOTS_PER_EPOCH=${SLOTS_PER_EPOCH:-6} " # Spec default: 32
DEFS+="-d:SECONDS_PER_SLOT=${SECONDS_PER_SLOT:-6} " # Spec default: 12
LAST_VALIDATOR_NUM=$(( NUM_VALIDATORS - 1 ))

View File

@ -56,7 +56,8 @@ suite "[Unit - Spec - Block processing] Deposits " & preset():
# State transition
# ----------------------------------------
check: state.process_deposit(deposit, {skipValidation})
check: state.process_deposit(deposit,
{skipValidation, skipMerkleValidation})
# Check invariants
# ----------------------------------------
@ -100,7 +101,8 @@ suite "[Unit - Spec - Block processing] Deposits " & preset():
# State transition
# ----------------------------------------
check: state.process_deposit(deposit, {skipValidation})
check: state.process_deposit(deposit,
{skipValidation, skipMerkleValidation})
# Check invariants
# ----------------------------------------

View File

@ -111,7 +111,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
attestation1 = makeAttestation(
state.data.data, state.blck.root, bc0[1], cache)
attestation0.combine(attestation1, {skipValidation})
attestation0.combine(attestation1, {})
pool.add(attestation0)
pool.add(attestation1)
@ -135,7 +135,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
attestation1 = makeAttestation(
state.data.data, state.blck.root, bc0[1], cache)
attestation0.combine(attestation1, {skipValidation})
attestation0.combine(attestation1, {})
pool.add(attestation1)
pool.add(attestation0)

View File

@ -17,5 +17,5 @@ suite "Beacon state" & preset():
timedTest "Smoke test initialize_beacon_state_from_eth1" & preset():
let state = initialize_beacon_state_from_eth1(
Eth2Digest(), 0,
makeInitialDeposits(SLOTS_PER_EPOCH, {}), {skipValidation})
makeInitialDeposits(SLOTS_PER_EPOCH, {}), {skipMerkleValidation})
check: state.validators.len == SLOTS_PER_EPOCH

View File

@ -187,6 +187,55 @@ when const_preset == "minimal": # Too much stack space used on mainnet
pool.head.blck == b1Add
pool.headState.data.data.slot == b1Add.slot
timedTest "updateStateData sanity" & preset():
let
b1Add = pool.add(b1Root, b1)
b2Add = pool.add(b2Root, b2)
bs1 = BlockSlot(blck: b1Add, slot: b1.message.slot)
bs1_3 = b1Add.atSlot(3.Slot)
bs2 = BlockSlot(blck: b2Add, slot: b2.message.slot)
bs2_3 = b2Add.atSlot(3.Slot)
var tmpState = pool.headState
# move to specific block
pool.updateStateData(tmpState, bs1)
check:
tmpState.blck == b1Add
tmpState.data.data.slot == bs1.slot
# Skip slots
pool.updateStateData(tmpState, bs1_3) # skip slots
check:
tmpState.blck == b1Add
tmpState.data.data.slot == bs1_3.slot
# Move back slots, but not blocks
pool.updateStateData(tmpState, bs1_3.parent())
check:
tmpState.blck == b1Add
tmpState.data.data.slot == bs1_3.parent().slot
# Move to different block and slot
pool.updateStateData(tmpState, bs2_3)
check:
tmpState.blck == b2Add
tmpState.data.data.slot == bs2_3.slot
# Move back slot and block
pool.updateStateData(tmpState, bs1)
check:
tmpState.blck == b1Add
tmpState.data.data.slot == bs1.slot
# Move back to genesis
pool.updateStateData(tmpState, bs1.parent())
check:
tmpState.blck == b1Add.parent
tmpState.data.data.slot == bs1.parent.slot
suite "BlockPool finalization tests" & preset():
setup:
var
@ -213,7 +262,7 @@ when const_preset == "minimal": # Too much stack space used on mainnet
BeaconBlockBody(
attestations: makeFullAttestations(
pool.headState.data.data, pool.head.blck.root,
pool.headState.data.data.slot, cache, {skipValidation})))
pool.headState.data.data.slot, cache, {})))
let added = pool.add(hash_tree_root(blck.message), blck)
pool.updateHead(added)

View File

@ -150,7 +150,7 @@ suite "Interop":
var
initialState = initialize_beacon_state_from_eth1(
eth1BlockHash, 1570500000, deposits, {skipValidation})
eth1BlockHash, 1570500000, deposits, {skipMerkleValidation})
# https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start#create-genesis-state
initialState.genesis_time = 1570500000

View File

@ -22,7 +22,7 @@ suite "Block processing" & preset():
# TODO bls verification is a bit of a bottleneck here
genesisState = initialize_beacon_state_from_eth1(
Eth2Digest(), 0,
makeInitialDeposits(), {skipValidation})
makeInitialDeposits(), {skipMerkleValidation})
genesisBlock = get_initial_beacon_block(genesisState)
genesisRoot = hash_tree_root(genesisBlock.message)

View File

@ -80,7 +80,8 @@ proc makeTestDB*(validators: int): BeaconChainDB =
let
genState = initialize_beacon_state_from_eth1(
Eth2Digest(), 0,
makeInitialDeposits(validators, flags = {skipValidation}), {skipValidation})
makeInitialDeposits(validators, flags = {skipValidation}),
{skipValidation, skipMerkleValidation})
genBlock = get_initial_beacon_block(genState)
makeTestDB(genState, genBlock)

2
vendor/NimYAML vendored

@ -1 +1 @@
Subproject commit e6ac75d86af8f8eae9a010c8e639c3ac4263f8df
Subproject commit 3983a1bf48682156cb105ab124c7bbdd86fa4557

2
vendor/nim-libp2p vendored

@ -1 +1 @@
Subproject commit d42833947a4baddf21da8ac3105e2d5956a6daac
Subproject commit 8c406fb9e5f5ea82c96173b828efe4a8a6027747

@ -1 +1 @@
Subproject commit 2a70c4f152ee849db1ededa92c1d80f7102dd718
Subproject commit 2c4faa5372d2d8b0c2d16710fe5f93beab1c86af