Merge pull request #623 from status-im/devel

[Hotfix] build instructions
This commit is contained in:
zah 2019-12-04 01:35:46 +02:00 committed by GitHub
commit 013147464b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 282 additions and 246 deletions

58
Jenkinsfile vendored
View File

@ -1,26 +1,42 @@
def runStages() {
stage("Clone") {
/* The Git repo seems to be cached in some Jenkins plugin, so this is not always a clean clone. */
checkout scm
sh "make build-system-checks || true"
}
stage("Build") {
sh "make -j${env.NPROC} update" /* to allow a newer Nim version to be detected */
sh "make -j${env.NPROC} V=1 deps" /* to allow the following parallel stages */
}
stage("Test") {
parallel(
"tools": {
stage("Tools") {
sh "make -j${env.NPROC}"
}
},
"test suite": {
stage("Test suite") {
sh "make -j${env.NPROC} test"
}
try {
stage("Clone") {
checkout scm
sh "make build-system-checks || true"
}
cache(maxCacheSize: 250, caches: [
[$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/vendor/nimbus-build-system/vendor/Nim/bin"],
[$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/vendor/go/bin"],
[$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/jsonTestsCache"]
]) {
stage("Build") {
sh "make -j${env.NPROC} update" /* to allow a newer Nim version to be detected */
sh "make -j${env.NPROC} deps" /* to allow the following parallel stages */
sh "scripts/setup_official_tests.sh jsonTestsCache"
}
)
}
stage("Test") {
parallel(
"tools": {
stage("Tools") {
sh "make -j${env.NPROC}"
}
},
"test suite": {
stage("Test suite") {
sh "make -j${env.NPROC} DISABLE_TEST_FIXTURES_SCRIPT=1 test"
}
}
)
}
} catch(e) {
echo "'${env.STAGE_NAME}' stage failed"
// we need to rethrow the exception here
throw e
} finally {
cleanWs()
}
}

View File

@ -19,7 +19,12 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
.PHONY: all build-system-checks deps update p2pd test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation clean-testnet0 testnet0 clean-testnet1 testnet1 clean
ifeq ($(NIM_PARAMS),)
# "variables.mk" was not included. We can only execute one target in this state.
all: | build-system-checks
else
all: | build-system-checks $(TOOLS)
endif
# must be included after the default target
-include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk
@ -29,9 +34,11 @@ build-system-checks:
@[[ -e "$(BUILD_SYSTEM_DIR)/makefiles" ]] || { \
echo -e "'$(BUILD_SYSTEM_DIR)/makefiles' not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
$(GIT_SUBMODULE_UPDATE); \
echo -e "\nYou can now run '$(MAKE)' again."; \
exit 1; \
}
CHECKMARK="\xe2\x9c\x94\xef\xb8\x8f"; \
echo -e "\n$${CHECKMARK}$${CHECKMARK}$${CHECKMARK} Successfully fetched all required internal dependencies."; \
echo -e " You should now \e[4mre-run '$(MAKE)' to build Nimbus\e[0m\n"; \
}; \
exit 0
deps: | deps-common beacon_chain.nims p2pd

View File

@ -98,8 +98,8 @@ Once the [prerequisites](#prerequisites) are installed you can connect to testne
git clone https://github.com/status-im/nim-beacon-chain
cd nim-beacon-chain
make # This invocation will bootstrap the build system with additional Makefiles
make update deps # This will build Nim and all other dependencies
./connect-to-testnet testnet0
make testnet0 # This will build Nimbus and all other dependencies
# and connect you to testnet0
```
The testnets are restarted once per week, usually on Monday evenings (UTC)) and integrate the changes for the past week.

View File

@ -26,6 +26,7 @@ additional Nim arguments, e.g.:
```bash
make libnfuzz.a NIMFLAGS="--cc:clang --passC:'-fsanitize=fuzzer' --passL='-fsanitize=fuzzer'"
```
To disable BLS verification on deserialization of SSZ objects add `-d:ssz_testing` to the NIMFLAGS.
Other useful options might include: `--clang.path:<path>`, `--clang.exe:<exe>`, `--clang.linkerexe:<exe>`.

View File

@ -1,8 +1,8 @@
import
confutils, stats, times,
confutils, stats, times, std/monotimes,
strformat,
options, sequtils, random, tables,
../tests/[testutil],
../tests/[testutil, testblockutil],
../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator],
../beacon_chain/[attestation_pool, extras, ssz]
@ -13,24 +13,6 @@ type Timers = enum
tShuffle = "Retrieve committee once using get_beacon_committee"
tAttest = "Combine committee attestations"
template withTimer(stats: var RunningStat, body: untyped) =
let start = cpuTime()
block:
body
let stop = cpuTime()
stats.push stop - start
template withTimerRet(stats: var RunningStat, body: untyped): untyped =
let start = cpuTime()
let tmp = block:
body
let stop = cpuTime()
stats.push stop - start
tmp
proc writeJson*(prefix, slot, v: auto) =
var f: File
defer: close(f)
@ -54,7 +36,7 @@ func verifyConsensus(state: BeaconState, attesterRatio: auto) =
doAssert state.finalized_checkpoint.epoch + 2 >= current_epoch
cli do(slots = SLOTS_PER_EPOCH * 6,
validators = SLOTS_PER_EPOCH * 11, # One per shard is minimum
validators = SLOTS_PER_EPOCH * 30, # One per shard is minimum
json_interval = SLOTS_PER_EPOCH,
prefix = 0,
attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.73,

View File

@ -10,7 +10,7 @@
import
options, unittest,
chronicles,
./testutil,
./testutil, ./testblockutil,
../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator],
../beacon_chain/[beacon_node_types, attestation_pool, block_pool, extras, state_transition, ssz]

View File

@ -11,7 +11,7 @@ import options, unittest, sequtils, eth/trie/[db],
../beacon_chain/[beacon_chain_db, extras, interop, ssz],
../beacon_chain/spec/[beaconstate, datatypes, digest, crypto],
# test utilies
./testutil
./testutil, ./testblockutil
suite "Beacon chain DB" & preset():

View File

@ -9,7 +9,7 @@
import
unittest,
./testutil,
./testutil, ./testblockutil,
../beacon_chain/spec/[beaconstate, datatypes, digest]
suite "Beacon state" & preset():

View File

@ -9,7 +9,7 @@
import
options, sequtils, unittest,
./testutil,
./testutil, ./testblockutil,
../beacon_chain/spec/[beaconstate, datatypes, digest],
../beacon_chain/[beacon_node_types, block_pool, beacon_chain_db, extras, ssz]

View File

@ -9,7 +9,7 @@
import
unittest,
./testutil,
./testutil, ./testblockutil,
../beacon_chain/spec/[beaconstate, datatypes, digest, validator],
../beacon_chain/[state_transition, ssz]

207
tests/testblockutil.nim Normal file
View File

@ -0,0 +1,207 @@
# beacon_chain
# Copyright (c) 2018-2019 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
stew/endians2,
chronicles, eth/trie/[db],
../beacon_chain/[beacon_chain_db, block_pool, extras, ssz, state_transition,
validator_pool],
../beacon_chain/spec/[beaconstate, crypto, datatypes, digest,
helpers, validator]
when ValidatorPrivKey is BlsValue:
func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey =
# 0 is not a valid BLS private key - 1000 helps interop with rust BLS library,
# lighthouse.
# TODO: switch to https://github.com/ethereum/eth2.0-pm/issues/60
result.kind = BlsValueType.Real
var bytes = uint64(i + 1000).toBytesLE()
copyMem(addr result.blsValue.x[0], addr bytes[0], sizeof(bytes))
else:
func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey =
# 0 is not a valid BLS private key - 1000 helps interop with rust BLS library,
# lighthouse.
# TODO: switch to https://github.com/ethereum/eth2.0-pm/issues/60
var bytes = uint64(i + 1000).toBytesLE()
copyMem(addr result.x[0], addr bytes[0], sizeof(bytes))
func makeFakeHash(i: int): Eth2Digest =
var bytes = uint64(i).toBytesLE()
static: doAssert sizeof(bytes) <= sizeof(result.data)
copyMem(addr result.data[0], addr bytes[0], sizeof(bytes))
func hackPrivKey(v: Validator): ValidatorPrivKey =
## Extract private key, per above hack
var bytes: array[8, byte]
static: doAssert sizeof(bytes) <= sizeof(v.withdrawal_credentials.data)
copyMem(
addr bytes, unsafeAddr v.withdrawal_credentials.data[0], sizeof(bytes))
let i = int(uint64.fromBytesLE(bytes))
makeFakeValidatorPrivKey(i)
func makeDeposit(i: int, flags: UpdateFlags): Deposit =
## Ugly hack for now: we stick the private key in withdrawal_credentials
## which means we can repro private key and randao reveal from this data,
## for testing :)
let
privkey = makeFakeValidatorPrivKey(i)
pubkey = privkey.pubKey()
withdrawal_credentials = makeFakeHash(i)
domain = compute_domain(DOMAIN_DEPOSIT)
result = Deposit(
data: DepositData(
pubkey: pubkey,
withdrawal_credentials: withdrawal_credentials,
amount: MAX_EFFECTIVE_BALANCE,
)
)
if skipValidation notin flags:
result.data.signature =
bls_sign(privkey, signing_root(result.data).data,
domain)
func makeInitialDeposits*(
n = SLOTS_PER_EPOCH, flags: UpdateFlags = {}): seq[Deposit] =
for i in 0..<n.int:
result.add makeDeposit(i, flags)
proc addBlock*(
state: var BeaconState, previous_block_root: Eth2Digest,
body: BeaconBlockBody, flags: UpdateFlags = {}): BeaconBlock =
# Create and add a block to state - state will advance by one slot!
# This is the equivalent of running
# updateState(state, prev_block, makeBlock(...), {skipValidation})
# but avoids some slow block copies
state.slot += 1
var cache = get_empty_per_epoch_cache()
let proposer_index = get_beacon_proposer_index(state, cache)
state.slot -= 1
let
# Index from the new state, but registry from the old state.. hmm...
proposer = state.validators[proposer_index]
privKey = hackPrivKey(proposer)
# TODO ugly hack; API needs rethinking
var new_body = body
new_body.randao_reveal = privKey.genRandaoReveal(state.fork, state.slot + 1)
new_body.eth1_data = Eth1Data()
var
# In order to reuse the state transition function, we first create a dummy
# block that has some fields set, and use that to generate the state as it
# would look with the new block applied.
new_block = BeaconBlock(
slot: state.slot + 1,
parent_root: previous_block_root,
state_root: Eth2Digest(), # we need the new state first
body: new_body,
signature: ValidatorSig(), # we need the rest of the block first!
)
let block_ok = state_transition(state, new_block, {skipValidation})
doAssert block_ok
# Ok, we have the new state as it would look with the block applied - now we
# can set the state root in order to be able to create a valid signature
new_block.state_root = hash_tree_root(state)
doAssert privKey.pubKey() == proposer.pubkey,
"signature key should be derived from private key! - wrong privkey?"
if skipValidation notin flags:
let block_root = signing_root(new_block)
# We have a signature - put it in the block and we should be done!
new_block.signature =
bls_sign(privKey, block_root.data,
get_domain(state, DOMAIN_BEACON_PROPOSER,
compute_epoch_at_slot(new_block.slot)))
doAssert bls_verify(
proposer.pubkey,
block_root.data, new_block.signature,
get_domain(
state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(new_block.slot))),
"we just signed this message - it should pass verification!"
new_block
proc makeBlock*(
state: BeaconState, previous_block_root: Eth2Digest,
body: BeaconBlockBody): BeaconBlock =
# Create a block for `state.slot + 1` - like a block proposer would do!
# It's a bit awkward - in order to produce a block for N+1, we need to
# calculate what the state will look like after that block has been applied,
# because the block includes the state root.
var next_state = state
addBlock(next_state, previous_block_root, body)
proc makeAttestation*(
state: BeaconState, beacon_block_root: Eth2Digest,
committee: seq[ValidatorIndex], slot: Slot, index: uint64,
validator_index: auto, cache: var StateCache,
flags: UpdateFlags = {}): Attestation =
# Avoids state_sim silliness; as it's responsible for all validators,
# transforming, from monotonic enumerable index -> committee index ->
# montonoic enumerable index, is wasteful and slow. Most test callers
# want ValidatorIndex, so that's supported too.
let
validator = state.validators[validator_index]
sac_index = committee.find(validator_index)
data = makeAttestationData(state, slot, index, beacon_block_root)
doAssert sac_index != -1, "find_beacon_committee should guarantee this"
var aggregation_bits = CommitteeValidatorsBits.init(committee.len)
aggregation_bits.raiseBit sac_index
let
msg = hash_tree_root(data)
sig =
if skipValidation notin flags:
bls_sign(
hackPrivKey(validator), @(msg.data),
get_domain(
state,
DOMAIN_BEACON_ATTESTER,
data.target.epoch))
else:
ValidatorSig()
Attestation(
data: data,
aggregation_bits: aggregation_bits,
signature: sig
)
proc find_beacon_committee(
state: BeaconState, validator_index: ValidatorIndex,
cache: var StateCache): auto =
let epoch = compute_epoch_at_slot(state.slot)
for epoch_committee_index in 0'u64 ..< get_committee_count_at_slot(
state, epoch.compute_start_slot_at_epoch) * SLOTS_PER_EPOCH:
let
slot = ((epoch_committee_index mod SLOTS_PER_EPOCH) +
epoch.compute_start_slot_at_epoch.uint64).Slot
index = epoch_committee_index div SLOTS_PER_EPOCH
committee = get_beacon_committee(state, slot, index, cache)
if validator_index in committee:
return (committee, slot, index)
doAssert false
proc makeAttestation*(
state: BeaconState, beacon_block_root: Eth2Digest,
validator_index: ValidatorIndex, cache: var StateCache,
flags: UpdateFlags = {}): Attestation =
let (committee, slot, index) =
find_beacon_committee(state, validator_index, cache)
makeAttestation(state, beacon_block_root, committee, slot, index,
validator_index, cache, flags)

View File

@ -6,208 +6,31 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
options, stew/endians2,
stats, stew/endians2,
chronicles, eth/trie/[db],
../beacon_chain/[beacon_chain_db, block_pool, extras, ssz, state_transition,
validator_pool, beacon_node_types],
../beacon_chain/spec/[beaconstate, crypto, datatypes, digest,
helpers, validator]
../beacon_chain/[beacon_chain_db, block_pool, ssz, beacon_node_types],
../beacon_chain/spec/datatypes
func preset*(): string =
" [Preset: " & const_preset & ']'
when ValidatorPrivKey is BlsValue:
func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey =
# 0 is not a valid BLS private key - 1000 helps interop with rust BLS library,
# lighthouse.
# TODO: switch to https://github.com/ethereum/eth2.0-pm/issues/60
result.kind = BlsValueType.Real
var bytes = uint64(i + 1000).toBytesLE()
copyMem(addr result.blsValue.x[0], addr bytes[0], sizeof(bytes))
else:
func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey =
# 0 is not a valid BLS private key - 1000 helps interop with rust BLS library,
# lighthouse.
# TODO: switch to https://github.com/ethereum/eth2.0-pm/issues/60
var bytes = uint64(i + 1000).toBytesLE()
copyMem(addr result.x[0], addr bytes[0], sizeof(bytes))
template withTimer*(stats: var RunningStat, body: untyped) =
let start = getMonoTime()
func makeFakeHash(i: int): Eth2Digest =
var bytes = uint64(i).toBytesLE()
static: doAssert sizeof(bytes) <= sizeof(result.data)
copyMem(addr result.data[0], addr bytes[0], sizeof(bytes))
block:
body
func hackPrivKey(v: Validator): ValidatorPrivKey =
## Extract private key, per above hack
var bytes: array[8, byte]
static: doAssert sizeof(bytes) <= sizeof(v.withdrawal_credentials.data)
let stop = getMonoTime()
stats.push (stop - start).inMicroseconds.float / 1000000.0
copyMem(
addr bytes, unsafeAddr v.withdrawal_credentials.data[0], sizeof(bytes))
let i = int(uint64.fromBytesLE(bytes))
makeFakeValidatorPrivKey(i)
template withTimerRet*(stats: var RunningStat, body: untyped): untyped =
let start = getMonoTime()
let tmp = block:
body
let stop = getMonoTime()
stats.push (stop - start).inMicroseconds.float / 1000000.0
func makeDeposit(i: int, flags: UpdateFlags): Deposit =
## Ugly hack for now: we stick the private key in withdrawal_credentials
## which means we can repro private key and randao reveal from this data,
## for testing :)
let
privkey = makeFakeValidatorPrivKey(i)
pubkey = privkey.pubKey()
withdrawal_credentials = makeFakeHash(i)
domain = compute_domain(DOMAIN_DEPOSIT)
result = Deposit(
data: DepositData(
pubkey: pubkey,
withdrawal_credentials: withdrawal_credentials,
amount: MAX_EFFECTIVE_BALANCE,
)
)
if skipValidation notin flags:
result.data.signature =
bls_sign(privkey, signing_root(result.data).data,
domain)
func makeInitialDeposits*(
n = SLOTS_PER_EPOCH, flags: UpdateFlags = {}): seq[Deposit] =
for i in 0..<n.int:
result.add makeDeposit(i, flags)
proc addBlock*(
state: var BeaconState, previous_block_root: Eth2Digest,
body: BeaconBlockBody, flags: UpdateFlags = {}): BeaconBlock =
# Create and add a block to state - state will advance by one slot!
# This is the equivalent of running
# updateState(state, prev_block, makeBlock(...), {skipValidation})
# but avoids some slow block copies
state.slot += 1
var cache = get_empty_per_epoch_cache()
let proposer_index = get_beacon_proposer_index(state, cache)
state.slot -= 1
let
# Index from the new state, but registry from the old state.. hmm...
proposer = state.validators[proposer_index]
privKey = hackPrivKey(proposer)
# TODO ugly hack; API needs rethinking
var new_body = body
new_body.randao_reveal = privKey.genRandaoReveal(state.fork, state.slot + 1)
new_body.eth1_data = Eth1Data()
var
# In order to reuse the state transition function, we first create a dummy
# block that has some fields set, and use that to generate the state as it
# would look with the new block applied.
new_block = BeaconBlock(
slot: state.slot + 1,
parent_root: previous_block_root,
state_root: Eth2Digest(), # we need the new state first
body: new_body,
signature: ValidatorSig(), # we need the rest of the block first!
)
let block_ok = state_transition(state, new_block, {skipValidation})
doAssert block_ok
# Ok, we have the new state as it would look with the block applied - now we
# can set the state root in order to be able to create a valid signature
new_block.state_root = hash_tree_root(state)
doAssert privKey.pubKey() == proposer.pubkey,
"signature key should be derived from private key! - wrong privkey?"
if skipValidation notin flags:
let block_root = signing_root(new_block)
# We have a signature - put it in the block and we should be done!
new_block.signature =
bls_sign(privKey, block_root.data,
get_domain(state, DOMAIN_BEACON_PROPOSER,
compute_epoch_at_slot(new_block.slot)))
doAssert bls_verify(
proposer.pubkey,
block_root.data, new_block.signature,
get_domain(
state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(new_block.slot))),
"we just signed this message - it should pass verification!"
new_block
proc makeBlock*(
state: BeaconState, previous_block_root: Eth2Digest,
body: BeaconBlockBody): BeaconBlock =
# Create a block for `state.slot + 1` - like a block proposer would do!
# It's a bit awkward - in order to produce a block for N+1, we need to
# calculate what the state will look like after that block has been applied,
# because the block includes the state root.
var next_state = state
addBlock(next_state, previous_block_root, body)
proc find_beacon_committee(
state: BeaconState, validator_index: ValidatorIndex,
cache: var StateCache): auto =
let epoch = compute_epoch_at_slot(state.slot)
for epoch_committee_index in 0'u64 ..< get_committee_count_at_slot(
state, epoch.compute_start_slot_at_epoch) * SLOTS_PER_EPOCH:
let
slot = ((epoch_committee_index mod SLOTS_PER_EPOCH) +
epoch.compute_start_slot_at_epoch.uint64).Slot
index = epoch_committee_index div SLOTS_PER_EPOCH
committee = get_beacon_committee(state, slot, index, cache)
if validator_index in committee:
return (committee, slot, index)
doAssert false
proc makeAttestation*(
state: BeaconState, beacon_block_root: Eth2Digest,
committee: seq[ValidatorIndex], slot: Slot, index: uint64,
validator_index: auto, cache: var StateCache,
flags: UpdateFlags = {}): Attestation =
# Avoids state_sim silliness; as it's responsible for all validators,
# transforming, from monotonic enumerable index -> committee index ->
# montonoic enumerable index, is wasteful and slow. Most test callers
# want ValidatorIndex, so that's supported too.
let
validator = state.validators[validator_index]
sac_index = committee.find(validator_index)
data = makeAttestationData(state, slot, index, beacon_block_root)
doAssert sac_index != -1, "find_beacon_committee should guarantee this"
var aggregation_bits = CommitteeValidatorsBits.init(committee.len)
aggregation_bits.raiseBit sac_index
let
msg = hash_tree_root(data)
sig =
if skipValidation notin flags:
bls_sign(
hackPrivKey(validator), @(msg.data),
get_domain(
state,
DOMAIN_BEACON_ATTESTER,
data.target.epoch))
else:
ValidatorSig()
Attestation(
data: data,
aggregation_bits: aggregation_bits,
signature: sig
)
proc makeAttestation*(
state: BeaconState, beacon_block_root: Eth2Digest,
validator_index: ValidatorIndex, cache: var StateCache,
flags: UpdateFlags = {}): Attestation =
let (committee, slot, index) =
find_beacon_committee(state, validator_index, cache)
makeAttestation(state, beacon_block_root, committee, slot, index,
validator_index, cache, flags)
tmp
proc makeTestDB*(tailState: BeaconState, tailBlock: BeaconBlock): BeaconChainDB =
result = init(BeaconChainDB, newMemoryDB())

@ -1 +1 @@
Subproject commit 42c5e97a5ba2cf31a573160e7d0a2efd615334c5
Subproject commit 6cfabf7820834cb99bd30fc664d3ab91eb0493bf