diff --git a/Jenkinsfile b/Jenkinsfile index f8da394af..4055335a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,26 +1,42 @@ def runStages() { - stage("Clone") { - /* The Git repo seems to be cached in some Jenkins plugin, so this is not always a clean clone. */ - checkout scm - sh "make build-system-checks || true" - } - stage("Build") { - sh "make -j${env.NPROC} update" /* to allow a newer Nim version to be detected */ - sh "make -j${env.NPROC} V=1 deps" /* to allow the following parallel stages */ - } - stage("Test") { - parallel( - "tools": { - stage("Tools") { - sh "make -j${env.NPROC}" - } - }, - "test suite": { - stage("Test suite") { - sh "make -j${env.NPROC} test" - } + try { + stage("Clone") { + checkout scm + sh "make build-system-checks || true" + } + + cache(maxCacheSize: 250, caches: [ + [$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/vendor/nimbus-build-system/vendor/Nim/bin"], + [$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/vendor/go/bin"], + [$class: "ArbitraryFileCache", excludes: "", includes: "**/*", path: "${WORKSPACE}/jsonTestsCache"] + ]) { + stage("Build") { + sh "make -j${env.NPROC} update" /* to allow a newer Nim version to be detected */ + sh "make -j${env.NPROC} deps" /* to allow the following parallel stages */ + sh "scripts/setup_official_tests.sh jsonTestsCache" } - ) + } + + stage("Test") { + parallel( + "tools": { + stage("Tools") { + sh "make -j${env.NPROC}" + } + }, + "test suite": { + stage("Test suite") { + sh "make -j${env.NPROC} DISABLE_TEST_FIXTURES_SCRIPT=1 test" + } + } + ) + } + } catch(e) { + echo "'${env.STAGE_NAME}' stage failed" + // we need to rethrow the exception here + throw e + } finally { + cleanWs() } } diff --git a/Makefile b/Makefile index e3ada862f..77c2e833e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,12 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) .PHONY: all build-system-checks deps update p2pd test $(TOOLS) clean_eth2_network_simulation_files eth2_network_simulation clean-testnet0 testnet0 clean-testnet1 testnet1 clean +ifeq ($(NIM_PARAMS),) +# "variables.mk" was not included. We can only execute one target in this state. +all: | build-system-checks +else all: | build-system-checks $(TOOLS) +endif # must be included after the default target -include $(BUILD_SYSTEM_DIR)/makefiles/targets.mk @@ -29,9 +34,11 @@ build-system-checks: @[[ -e "$(BUILD_SYSTEM_DIR)/makefiles" ]] || { \ echo -e "'$(BUILD_SYSTEM_DIR)/makefiles' not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \ $(GIT_SUBMODULE_UPDATE); \ - echo -e "\nYou can now run '$(MAKE)' again."; \ - exit 1; \ - } + CHECKMARK="\xe2\x9c\x94\xef\xb8\x8f"; \ + echo -e "\n$${CHECKMARK}$${CHECKMARK}$${CHECKMARK} Successfully fetched all required internal dependencies."; \ + echo -e " You should now \e[4mre-run '$(MAKE)' to build Nimbus\e[0m\n"; \ + }; \ + exit 0 deps: | deps-common beacon_chain.nims p2pd diff --git a/README.md b/README.md index 465103551..d93c522da 100644 --- a/README.md +++ b/README.md @@ -98,8 +98,8 @@ Once the [prerequisites](#prerequisites) are installed you can connect to testne git clone https://github.com/status-im/nim-beacon-chain cd nim-beacon-chain make # This invocation will bootstrap the build system with additional Makefiles -make update deps # This will build Nim and all other dependencies -./connect-to-testnet testnet0 +make testnet0 # This will build Nimbus and all other dependencies + # and connect you to testnet0 ``` The testnets are restarted once per week, usually on Monday evenings (UTC)) and integrate the changes for the past week. diff --git a/nfuzz/README.md b/nfuzz/README.md index aa9df3e05..8ca9e3a40 100644 --- a/nfuzz/README.md +++ b/nfuzz/README.md @@ -26,6 +26,7 @@ additional Nim arguments, e.g.: ```bash make libnfuzz.a NIMFLAGS="--cc:clang --passC:'-fsanitize=fuzzer' --passL='-fsanitize=fuzzer'" ``` +To disable BLS verification on deserialization of SSZ objects add `-d:ssz_testing` to the NIMFLAGS. Other useful options might include: `--clang.path:`, `--clang.exe:`, `--clang.linkerexe:`. diff --git a/research/state_sim.nim b/research/state_sim.nim index e7dc2be63..9acdb2981 100644 --- a/research/state_sim.nim +++ b/research/state_sim.nim @@ -1,8 +1,8 @@ import - confutils, stats, times, + confutils, stats, times, std/monotimes, strformat, options, sequtils, random, tables, - ../tests/[testutil], + ../tests/[testutil, testblockutil], ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], ../beacon_chain/[attestation_pool, extras, ssz] @@ -13,24 +13,6 @@ type Timers = enum tShuffle = "Retrieve committee once using get_beacon_committee" tAttest = "Combine committee attestations" -template withTimer(stats: var RunningStat, body: untyped) = - let start = cpuTime() - - block: - body - - let stop = cpuTime() - stats.push stop - start - -template withTimerRet(stats: var RunningStat, body: untyped): untyped = - let start = cpuTime() - let tmp = block: - body - let stop = cpuTime() - stats.push stop - start - - tmp - proc writeJson*(prefix, slot, v: auto) = var f: File defer: close(f) @@ -54,7 +36,7 @@ func verifyConsensus(state: BeaconState, attesterRatio: auto) = doAssert state.finalized_checkpoint.epoch + 2 >= current_epoch cli do(slots = SLOTS_PER_EPOCH * 6, - validators = SLOTS_PER_EPOCH * 11, # One per shard is minimum + validators = SLOTS_PER_EPOCH * 30, # One per shard is minimum json_interval = SLOTS_PER_EPOCH, prefix = 0, attesterRatio {.desc: "ratio of validators that attest in each round"} = 0.73, diff --git a/tests/test_attestation_pool.nim b/tests/test_attestation_pool.nim index 050238a0d..aca42ca79 100644 --- a/tests/test_attestation_pool.nim +++ b/tests/test_attestation_pool.nim @@ -10,7 +10,7 @@ import options, unittest, chronicles, - ./testutil, + ./testutil, ./testblockutil, ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, helpers, validator], ../beacon_chain/[beacon_node_types, attestation_pool, block_pool, extras, state_transition, ssz] diff --git a/tests/test_beacon_chain_db.nim b/tests/test_beacon_chain_db.nim index 35bbfa4fe..d6858f0bb 100644 --- a/tests/test_beacon_chain_db.nim +++ b/tests/test_beacon_chain_db.nim @@ -11,7 +11,7 @@ import options, unittest, sequtils, eth/trie/[db], ../beacon_chain/[beacon_chain_db, extras, interop, ssz], ../beacon_chain/spec/[beaconstate, datatypes, digest, crypto], # test utilies - ./testutil + ./testutil, ./testblockutil suite "Beacon chain DB" & preset(): diff --git a/tests/test_beaconstate.nim b/tests/test_beaconstate.nim index 4cb3be36d..09ca42bfb 100644 --- a/tests/test_beaconstate.nim +++ b/tests/test_beaconstate.nim @@ -9,7 +9,7 @@ import unittest, - ./testutil, + ./testutil, ./testblockutil, ../beacon_chain/spec/[beaconstate, datatypes, digest] suite "Beacon state" & preset(): diff --git a/tests/test_block_pool.nim b/tests/test_block_pool.nim index b1b154c27..3e70a006d 100644 --- a/tests/test_block_pool.nim +++ b/tests/test_block_pool.nim @@ -9,7 +9,7 @@ import options, sequtils, unittest, - ./testutil, + ./testutil, ./testblockutil, ../beacon_chain/spec/[beaconstate, datatypes, digest], ../beacon_chain/[beacon_node_types, block_pool, beacon_chain_db, extras, ssz] diff --git a/tests/test_state_transition.nim b/tests/test_state_transition.nim index 10a2e5c82..ec4a47071 100644 --- a/tests/test_state_transition.nim +++ b/tests/test_state_transition.nim @@ -9,7 +9,7 @@ import unittest, - ./testutil, + ./testutil, ./testblockutil, ../beacon_chain/spec/[beaconstate, datatypes, digest, validator], ../beacon_chain/[state_transition, ssz] diff --git a/tests/testblockutil.nim b/tests/testblockutil.nim new file mode 100644 index 000000000..d59bc7182 --- /dev/null +++ b/tests/testblockutil.nim @@ -0,0 +1,207 @@ +# beacon_chain +# Copyright (c) 2018-2019 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + +import + stew/endians2, + chronicles, eth/trie/[db], + ../beacon_chain/[beacon_chain_db, block_pool, extras, ssz, state_transition, + validator_pool], + ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, + helpers, validator] + +when ValidatorPrivKey is BlsValue: + func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey = + # 0 is not a valid BLS private key - 1000 helps interop with rust BLS library, + # lighthouse. + # TODO: switch to https://github.com/ethereum/eth2.0-pm/issues/60 + result.kind = BlsValueType.Real + var bytes = uint64(i + 1000).toBytesLE() + copyMem(addr result.blsValue.x[0], addr bytes[0], sizeof(bytes)) +else: + func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey = + # 0 is not a valid BLS private key - 1000 helps interop with rust BLS library, + # lighthouse. + # TODO: switch to https://github.com/ethereum/eth2.0-pm/issues/60 + var bytes = uint64(i + 1000).toBytesLE() + copyMem(addr result.x[0], addr bytes[0], sizeof(bytes)) + +func makeFakeHash(i: int): Eth2Digest = + var bytes = uint64(i).toBytesLE() + static: doAssert sizeof(bytes) <= sizeof(result.data) + copyMem(addr result.data[0], addr bytes[0], sizeof(bytes)) + +func hackPrivKey(v: Validator): ValidatorPrivKey = + ## Extract private key, per above hack + var bytes: array[8, byte] + static: doAssert sizeof(bytes) <= sizeof(v.withdrawal_credentials.data) + + copyMem( + addr bytes, unsafeAddr v.withdrawal_credentials.data[0], sizeof(bytes)) + let i = int(uint64.fromBytesLE(bytes)) + makeFakeValidatorPrivKey(i) + +func makeDeposit(i: int, flags: UpdateFlags): Deposit = + ## Ugly hack for now: we stick the private key in withdrawal_credentials + ## which means we can repro private key and randao reveal from this data, + ## for testing :) + let + privkey = makeFakeValidatorPrivKey(i) + pubkey = privkey.pubKey() + withdrawal_credentials = makeFakeHash(i) + domain = compute_domain(DOMAIN_DEPOSIT) + + result = Deposit( + data: DepositData( + pubkey: pubkey, + withdrawal_credentials: withdrawal_credentials, + amount: MAX_EFFECTIVE_BALANCE, + ) + ) + + if skipValidation notin flags: + result.data.signature = + bls_sign(privkey, signing_root(result.data).data, + domain) + +func makeInitialDeposits*( + n = SLOTS_PER_EPOCH, flags: UpdateFlags = {}): seq[Deposit] = + for i in 0.. committee index -> + # montonoic enumerable index, is wasteful and slow. Most test callers + # want ValidatorIndex, so that's supported too. + let + validator = state.validators[validator_index] + sac_index = committee.find(validator_index) + data = makeAttestationData(state, slot, index, beacon_block_root) + + doAssert sac_index != -1, "find_beacon_committee should guarantee this" + + var aggregation_bits = CommitteeValidatorsBits.init(committee.len) + aggregation_bits.raiseBit sac_index + + let + msg = hash_tree_root(data) + sig = + if skipValidation notin flags: + bls_sign( + hackPrivKey(validator), @(msg.data), + get_domain( + state, + DOMAIN_BEACON_ATTESTER, + data.target.epoch)) + else: + ValidatorSig() + + Attestation( + data: data, + aggregation_bits: aggregation_bits, + signature: sig + ) + +proc find_beacon_committee( + state: BeaconState, validator_index: ValidatorIndex, + cache: var StateCache): auto = + let epoch = compute_epoch_at_slot(state.slot) + for epoch_committee_index in 0'u64 ..< get_committee_count_at_slot( + state, epoch.compute_start_slot_at_epoch) * SLOTS_PER_EPOCH: + let + slot = ((epoch_committee_index mod SLOTS_PER_EPOCH) + + epoch.compute_start_slot_at_epoch.uint64).Slot + index = epoch_committee_index div SLOTS_PER_EPOCH + committee = get_beacon_committee(state, slot, index, cache) + if validator_index in committee: + return (committee, slot, index) + doAssert false + +proc makeAttestation*( + state: BeaconState, beacon_block_root: Eth2Digest, + validator_index: ValidatorIndex, cache: var StateCache, + flags: UpdateFlags = {}): Attestation = + let (committee, slot, index) = + find_beacon_committee(state, validator_index, cache) + makeAttestation(state, beacon_block_root, committee, slot, index, + validator_index, cache, flags) diff --git a/tests/testutil.nim b/tests/testutil.nim index dcdc6eeac..10a8c577b 100644 --- a/tests/testutil.nim +++ b/tests/testutil.nim @@ -6,208 +6,31 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import - options, stew/endians2, + stats, stew/endians2, chronicles, eth/trie/[db], - ../beacon_chain/[beacon_chain_db, block_pool, extras, ssz, state_transition, - validator_pool, beacon_node_types], - ../beacon_chain/spec/[beaconstate, crypto, datatypes, digest, - helpers, validator] + ../beacon_chain/[beacon_chain_db, block_pool, ssz, beacon_node_types], + ../beacon_chain/spec/datatypes func preset*(): string = " [Preset: " & const_preset & ']' -when ValidatorPrivKey is BlsValue: - func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey = - # 0 is not a valid BLS private key - 1000 helps interop with rust BLS library, - # lighthouse. - # TODO: switch to https://github.com/ethereum/eth2.0-pm/issues/60 - result.kind = BlsValueType.Real - var bytes = uint64(i + 1000).toBytesLE() - copyMem(addr result.blsValue.x[0], addr bytes[0], sizeof(bytes)) -else: - func makeFakeValidatorPrivKey(i: int): ValidatorPrivKey = - # 0 is not a valid BLS private key - 1000 helps interop with rust BLS library, - # lighthouse. - # TODO: switch to https://github.com/ethereum/eth2.0-pm/issues/60 - var bytes = uint64(i + 1000).toBytesLE() - copyMem(addr result.x[0], addr bytes[0], sizeof(bytes)) +template withTimer*(stats: var RunningStat, body: untyped) = + let start = getMonoTime() -func makeFakeHash(i: int): Eth2Digest = - var bytes = uint64(i).toBytesLE() - static: doAssert sizeof(bytes) <= sizeof(result.data) - copyMem(addr result.data[0], addr bytes[0], sizeof(bytes)) + block: + body -func hackPrivKey(v: Validator): ValidatorPrivKey = - ## Extract private key, per above hack - var bytes: array[8, byte] - static: doAssert sizeof(bytes) <= sizeof(v.withdrawal_credentials.data) + let stop = getMonoTime() + stats.push (stop - start).inMicroseconds.float / 1000000.0 - copyMem( - addr bytes, unsafeAddr v.withdrawal_credentials.data[0], sizeof(bytes)) - let i = int(uint64.fromBytesLE(bytes)) - makeFakeValidatorPrivKey(i) +template withTimerRet*(stats: var RunningStat, body: untyped): untyped = + let start = getMonoTime() + let tmp = block: + body + let stop = getMonoTime() + stats.push (stop - start).inMicroseconds.float / 1000000.0 -func makeDeposit(i: int, flags: UpdateFlags): Deposit = - ## Ugly hack for now: we stick the private key in withdrawal_credentials - ## which means we can repro private key and randao reveal from this data, - ## for testing :) - let - privkey = makeFakeValidatorPrivKey(i) - pubkey = privkey.pubKey() - withdrawal_credentials = makeFakeHash(i) - domain = compute_domain(DOMAIN_DEPOSIT) - - result = Deposit( - data: DepositData( - pubkey: pubkey, - withdrawal_credentials: withdrawal_credentials, - amount: MAX_EFFECTIVE_BALANCE, - ) - ) - - if skipValidation notin flags: - result.data.signature = - bls_sign(privkey, signing_root(result.data).data, - domain) - -func makeInitialDeposits*( - n = SLOTS_PER_EPOCH, flags: UpdateFlags = {}): seq[Deposit] = - for i in 0.. committee index -> - # montonoic enumerable index, is wasteful and slow. Most test callers - # want ValidatorIndex, so that's supported too. - let - validator = state.validators[validator_index] - sac_index = committee.find(validator_index) - data = makeAttestationData(state, slot, index, beacon_block_root) - - doAssert sac_index != -1, "find_beacon_committee should guarantee this" - - var aggregation_bits = CommitteeValidatorsBits.init(committee.len) - aggregation_bits.raiseBit sac_index - - let - msg = hash_tree_root(data) - sig = - if skipValidation notin flags: - bls_sign( - hackPrivKey(validator), @(msg.data), - get_domain( - state, - DOMAIN_BEACON_ATTESTER, - data.target.epoch)) - else: - ValidatorSig() - - Attestation( - data: data, - aggregation_bits: aggregation_bits, - signature: sig - ) - -proc makeAttestation*( - state: BeaconState, beacon_block_root: Eth2Digest, - validator_index: ValidatorIndex, cache: var StateCache, - flags: UpdateFlags = {}): Attestation = - let (committee, slot, index) = - find_beacon_committee(state, validator_index, cache) - makeAttestation(state, beacon_block_root, committee, slot, index, - validator_index, cache, flags) + tmp proc makeTestDB*(tailState: BeaconState, tailBlock: BeaconBlock): BeaconChainDB = result = init(BeaconChainDB, newMemoryDB()) diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index 42c5e97a5..6cfabf782 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit 42c5e97a5ba2cf31a573160e7d0a2efd615334c5 +Subproject commit 6cfabf7820834cb99bd30fc664d3ab91eb0493bf