diff --git a/.travis.yml b/.travis.yml index d16df789e..aca856766 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,4 +53,3 @@ script: - make -j${NPROC} NIMFLAGS="--parallelBuild:2" LOG_LEVEL=TRACE - make -j${NPROC} NIMFLAGS="--parallelBuild:2 -d:testnet_servers_image" LOG_LEVEL=TRACE beacon_node - make -j${NPROC} NIMFLAGS="--parallelBuild:2" DISABLE_TEST_FIXTURES_SCRIPT=1 test - diff --git a/Jenkinsfile b/Jenkinsfile index 6347c4236..e0277fc99 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -51,6 +51,14 @@ def runStages() { ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 """ } + // stage("testnet finalization - Miracl/Milagro fallback") { + // // EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node + // sh """#!/bin/bash + // set -e + // NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 + // NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5 + // """ + // } } ) } @@ -92,4 +100,3 @@ parallel( } } ) - diff --git a/beacon_chain.nimble b/beacon_chain.nimble index c1c93fd9a..89035feb7 100644 --- a/beacon_chain.nimble +++ b/beacon_chain.nimble @@ -59,21 +59,32 @@ task test, "Run all tests": # price we pay for that. # Just the part of minimal config which explicitly differs from mainnet - buildAndRunBinary "test_fixture_const_sanity_check", "tests/official/", "-d:const_preset=minimal" + buildAndRunBinary "test_fixture_const_sanity_check", "tests/official/", """-d:const_preset=minimal -d:chronicles_sinks="json[file]"""" # Mainnet config - buildAndRunBinary "proto_array", "beacon_chain/fork_choice/", "-d:const_preset=mainnet" - buildAndRunBinary "fork_choice", "beacon_chain/fork_choice/", "-d:const_preset=mainnet" - buildAndRunBinary "all_tests", "tests/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet" + buildAndRunBinary "proto_array", "beacon_chain/fork_choice/", """-d:const_preset=mainnet -d:chronicles_sinks="json[file]"""" + buildAndRunBinary "fork_choice", "beacon_chain/fork_choice/", """-d:const_preset=mainnet -d:chronicles_sinks="json[file]"""" + buildAndRunBinary "all_tests", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]"""" + + # Check Miracl/Milagro fallback on select tests + buildAndRunBinary "test_interop", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]"""" + buildAndRunBinary "test_process_attestation", "tests/spec_block_processing/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]"""" + buildAndRunBinary "test_process_deposits", "tests/spec_block_processing/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]"""" + buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]"""" + buildAndRunBinary "test_attestation_pool", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]"""" + buildAndRunBinary "test_block_pool", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]"""" # Generic SSZ test, doesn't use consensus objects minimal/mainnet presets - buildAndRunBinary "test_fixture_ssz_generic_types", "tests/official/", "-d:chronicles_log_level=TRACE" + buildAndRunBinary "test_fixture_ssz_generic_types", "tests/official/", """-d:chronicles_log_level=TRACE -d:chronicles_sinks="json[file]"""" # Consensus object SSZ tests - buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet" + buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]"""" - buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet" + # 0.12.1 + buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]"""" # State and block sims; getting to 4th epoch triggers consensus checks - buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet", "--validators=3000 --slots=128" + buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet -d:chronicles_log_level=INFO", "--validators=3000 --slots=128" + # buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_log_level=INFO", "--validators=3000 --slots=128" buildAndRunBinary "block_sim", "research/", "-d:const_preset=mainnet", "--validators=3000 --slots=128" + # buildAndRunBinary "block_sim", "research/", "-d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl", "--validators=3000 --slots=128" diff --git a/beacon_chain/attestation_pool.nim b/beacon_chain/attestation_pool.nim index fe5547929..3f7284adc 100644 --- a/beacon_chain/attestation_pool.nim +++ b/beacon_chain/attestation_pool.nim @@ -283,6 +283,9 @@ proc getAttestationsForBlock*(pool: AttestationPool, signature: a.validations[0].aggregate_signature ) + agg {.noInit.}: AggregateSignature + agg.init(a.validations[0].aggregate_signature) + # TODO what's going on here is that when producing a block, we need to # include only such attestations that will not cause block validation # to fail. How this interacts with voting and the acceptance of @@ -308,8 +311,9 @@ proc getAttestationsForBlock*(pool: AttestationPool, # one new attestation in there if not attestation.aggregation_bits.overlaps(v.aggregation_bits): attestation.aggregation_bits.combine(v.aggregation_bits) - attestation.signature.aggregate(v.aggregate_signature) + agg.aggregate(v.aggregate_signature) + attestation.signature = agg.finish() result.add(attestation) if result.lenu64 >= MAX_ATTESTATIONS: diff --git a/beacon_chain/spec/crypto.nim b/beacon_chain/spec/crypto.nim index b6d669b6c..3688a9f9b 100644 --- a/beacon_chain/spec/crypto.nim +++ b/beacon_chain/spec/crypto.nim @@ -44,7 +44,7 @@ export results, json_serialization const RawSigSize* = 96 RawPubKeySize* = 48 - RawPrivKeySize* = 48 + # RawPrivKeySize* = 48 for Miracl / 32 for BLST type BlsValueType* = enum @@ -77,6 +77,8 @@ type SomeSig* = TrustedSig | ValidatorSig +export AggregateSignature + func `==`*(a, b: BlsValue): bool = if a.kind != b.kind: return false if a.kind == Real: @@ -125,10 +127,20 @@ proc initPubKey*(pubkey: ValidatorPubKey): ValidatorPubKey = return ValidatorPubKey() key.get -func aggregate*(x: var ValidatorSig, other: ValidatorSig) = - ## Aggregate 2 Validator Signatures +func init*(agg: var AggregateSignature, sig: ValidatorSig) {.inline.}= + ## Initializes an aggregate signature context + ## This assumes that the signature is valid + agg.init(sig.blsValue) + +func aggregate*(agg: var AggregateSignature, sig: ValidatorSig) {.inline.}= + ## Aggregate two Validator Signatures ## This assumes that they are real signatures - x.blsValue.aggregate(other.blsValue) + agg.aggregate(sig.blsValue) + +func finish*(agg: AggregateSignature): ValidatorSig {.inline.}= + ## Canonicalize an AggregateSignature into a signature + result.kind = Real + result.blsValue.finish(agg) # https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/beacon-chain.md#bls-signatures proc blsVerify*( @@ -218,9 +230,14 @@ func `$`*(x: BlsValue): string = else: "raw: " & x.blob.toHex() -func toRaw*(x: ValidatorPrivKey): array[RawPrivKeySize, byte] = +func toRaw*(x: ValidatorPrivKey): array[32, byte] = # TODO: distinct type - see https://github.com/status-im/nim-blscurve/pull/67 - SecretKey(x).exportRaw() + when BLS_BACKEND == BLST: + result = SecretKey(x).exportRaw() + else: + # Miracl exports to 384-bit arrays, but Curve order is 256-bit + let raw = SecretKey(x).exportRaw() + result[0..32-1] = raw.toOpenArray(48-32, 48-1) func toRaw*(x: BlsValue): auto = if x.kind == Real: diff --git a/config.nims b/config.nims index cfc515007..6f12aac96 100644 --- a/config.nims +++ b/config.nims @@ -79,4 +79,3 @@ switch("warning", "LockLevel:off") # Useful for Chronos metrics. --define:chronosFutureTracking - diff --git a/research/simutils.nim b/research/simutils.nim index 0edaa129d..7cc115f36 100644 --- a/research/simutils.nim +++ b/research/simutils.nim @@ -99,19 +99,3 @@ proc printTimers*[Timers: enum]( echo "Validators: ", state.validators.len, ", epoch length: ", SLOTS_PER_EPOCH echo "Validators per attestation (mean): ", attesters.mean printTimers(validate, timers) - -proc combine*(tgt: var Attestation, src: Attestation, flags: UpdateFlags) = - ## Combine the signature and participation bitfield, with the assumption that - ## the same data is being signed - if the signatures overlap, they are not - ## combined. - - doAssert tgt.data == src.data - - # In a BLS aggregate signature, one needs to count how many times a - # particular public key has been added - since we use a single bit per key, we - # can only it once, thus we can never combine signatures that overlap already! - if not tgt.aggregation_bits.overlaps(src.aggregation_bits): - tgt.aggregation_bits.combine(src.aggregation_bits) - - if skipBlsValidation notin flags: - tgt.signature.aggregate(src.signature) diff --git a/research/state_sim.nim b/research/state_sim.nim index 7ef8a0a9d..71be1aa5b 100644 --- a/research/state_sim.nim +++ b/research/state_sim.nim @@ -124,18 +124,24 @@ cli do(slots = SLOTS_PER_EPOCH * 6, attesters.push scas.len() withTimer(timers[tAttest]): + var agg {.noInit.}: AggregateSignature for v in scas: if (rand(r, high(int)).float * attesterRatio).int <= high(int): if first: attestation = makeAttestation(state[].data, latest_block_root, scas, target_slot, i.uint64, v, cache, flags) + agg.init(attestation.signature) first = false else: - attestation.combine( + let att2 = makeAttestation(state[].data, latest_block_root, scas, target_slot, - i.uint64, v, cache, flags), - flags) + i.uint64, v, cache, flags) + if not att2.aggregation_bits.overlaps(attestation.aggregation_bits): + attestation.aggregation_bits.combine(att2.aggregation_bits) + if skipBlsValidation notin flags: + agg.aggregate(att2.signature) + attestation.signature = agg.finish() if not first: # add the attestation if any of the validators attested, as given diff --git a/scripts/launch_local_testnet.sh b/scripts/launch_local_testnet.sh index 2a1ffa04b..4c9b29007 100755 --- a/scripts/launch_local_testnet.sh +++ b/scripts/launch_local_testnet.sh @@ -160,7 +160,7 @@ else fi NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK}) -$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract +$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract PIDS="" WEB3_ARG="" @@ -347,4 +347,3 @@ else exit 1 fi fi - diff --git a/tests/helpers/debug_state.nim b/tests/helpers/debug_state.nim index f603571bf..f4c6e05dc 100644 --- a/tests/helpers/debug_state.nim +++ b/tests/helpers/debug_state.nim @@ -7,29 +7,10 @@ import macros, - nimcrypto/utils, ../../beacon_chain/spec/[datatypes, crypto, digest], ../../beacon_chain/ssz/types # digest is necessary for them to be printed as hex -# Define comparison of object variants for BLSValue -# https://github.com/nim-lang/Nim/issues/6676 -# (fully generic available - see also https://github.com/status-im/nim-beacon-chain/commit/993789bad684721bd7c74ea14b35c2d24dbb6e51) -# ---------------------------------------------------------------- - -proc `==`*(a, b: BlsValue): bool = - ## We sometimes need to compare real BlsValue - ## from parsed opaque blobs that are not really on the BLS curve - ## and full of zeros - if a.kind == Real: - if b.kind == Real: - a.blsvalue == b.blsValue - else: - $a.blsvalue == toHex(b.blob, true) - else: - if b.kind == Real: - toHex(a.blob, true) == $b.blsValue - else: - a.blob == b.blob +export crypto.`==` # --------------------------------------------------------------------- @@ -48,9 +29,10 @@ proc compareStmt(xSubField, ySubField: NimNode, stmts: var NimNode) = let xStr = $xSubField.toStrLit let yStr = $ySubField.toStrLit + let isEqual = bindSym("==") # Bind all expose equality, in particular for BlsValue stmts.add quote do: doAssert( - `xSubField` == `ySubField`, + `isEqual`(`xSubField`, `ySubField`), "\nDiff: " & `xStr` & " = " & $`xSubField` & "\n" & "and " & `yStr` & " = " & $`ySubField` & "\n" ) @@ -59,16 +41,16 @@ proc compareContainerStmt(xSubField, ySubField: NimNode, stmts: var NimNode) = let xStr = $xSubField.toStrLit let yStr = $ySubField.toStrLit - + let isEqual = bindSym("==") # Bind all expose equality, in particular for BlsValue stmts.add quote do: doAssert( - `xSubField`.len == `ySubField`.len, + `isEqual`(`xSubField`.len, `ySubField`.len), "\nDiff: " & `xStr` & ".len = " & $`xSubField`.len & "\n" & "and " & `yStr` & ".len = " & $`ySubField`.len & "\n" ) for idx in `xSubField`.low .. `xSubField`.high: doAssert( - `xSubField`[idx] == `ySubField`[idx], + `isEqual`(`xSubField`[idx], `ySubField`[idx]), "\nDiff: " & `xStr` & "[" & $idx & "] = " & $`xSubField`[idx] & "\n" & "and " & `yStr` & "[" & $idx & "] = " & $`ySubField`[idx] & "\n" ) diff --git a/tests/mocking/mock_attestations.nim b/tests/mocking/mock_attestations.nim index 5567bbfd8..b18aedca8 100644 --- a/tests/mocking/mock_attestations.nim +++ b/tests/mocking/mock_attestations.nim @@ -11,6 +11,8 @@ import # Standard library sets, + # Status + chronicles, # Specs ../../beacon_chain/spec/[datatypes, beaconstate, helpers, validator, crypto, signatures, state_transition, presets], @@ -63,6 +65,7 @@ proc signMockAttestation*(state: BeaconState, attestation: var Attestation) = cache ) + var agg {.noInit.}: AggregateSignature var first_iter = true # Can't do while loop on hashset for validator_index in participants: let sig = get_attestation_signature( @@ -70,10 +73,14 @@ proc signMockAttestation*(state: BeaconState, attestation: var Attestation) = MockPrivKeys[validator_index] ) if first_iter: - attestation.signature = sig + agg.init(sig) first_iter = false else: - aggregate(attestation.signature, sig) + agg.aggregate(sig) + + if first_iter != true: + attestation.signature = agg.finish() + # Otherwise no participants so zero sig proc mockAttestationImpl( state: BeaconState, diff --git a/tests/mocking/mock_validator_keys.nim b/tests/mocking/mock_validator_keys.nim index c478af389..050b20ac2 100644 --- a/tests/mocking/mock_validator_keys.nim +++ b/tests/mocking/mock_validator_keys.nim @@ -10,7 +10,7 @@ import bearssl, eth/keys, - blscurve/bls_signature_scheme, + blscurve, ../../beacon_chain/spec/[datatypes, crypto, presets] proc newKeyPair(rng: var BrHmacDrbgContext): BlsResult[tuple[pub: ValidatorPubKey, priv: ValidatorPrivKey]] = @@ -25,7 +25,7 @@ proc newKeyPair(rng: var BrHmacDrbgContext): BlsResult[tuple[pub: ValidatorPubKe var sk: SecretKey - pk: bls_signature_scheme.PublicKey + pk: blscurve.PublicKey if keyGen(ikm, pk, sk): ok((ValidatorPubKey(kind: Real, blsValue: pk), ValidatorPrivKey(sk))) else: diff --git a/tests/official/test_fixture_operations_block_header.nim b/tests/official/test_fixture_operations_block_header.nim index 5d2583a06..d9d5f1154 100644 --- a/tests/official/test_fixture_operations_block_header.nim +++ b/tests/official/test_fixture_operations_block_header.nim @@ -13,7 +13,7 @@ import # Utilities stew/results, # Beacon chain internals - ../../beacon_chain/spec/[datatypes, state_transition_block], + ../../beacon_chain/spec/[datatypes, state_transition_block, crypto], ../../beacon_chain/ssz, # Test utilities ../testutil, diff --git a/tests/test_attestation_pool.nim b/tests/test_attestation_pool.nim index 080fd0fcc..64e231d27 100644 --- a/tests/test_attestation_pool.nim +++ b/tests/test_attestation_pool.nim @@ -11,13 +11,32 @@ import unittest, chronicles, stew/byteutils, - ./testutil, ./testblockutil, ../research/simutils, + ./testutil, ./testblockutil, ../beacon_chain/spec/[crypto, datatypes, digest, validator, state_transition, helpers, beaconstate, presets], ../beacon_chain/[beacon_node_types, attestation_pool, extras], ../beacon_chain/fork_choice/[fork_choice_types, fork_choice], ../beacon_chain/block_pools/[chain_dag, clearance] +func combine(tgt: var Attestation, src: Attestation, flags: UpdateFlags) = + ## Combine the signature and participation bitfield, with the assumption that + ## the same data is being signed - if the signatures overlap, they are not + ## combined. + + doAssert tgt.data == src.data + + # In a BLS aggregate signature, one needs to count how many times a + # particular public key has been added - since we use a single bit per key, we + # can only it once, thus we can never combine signatures that overlap already! + if not tgt.aggregation_bits.overlaps(src.aggregation_bits): + tgt.aggregation_bits.combine(src.aggregation_bits) + + if skipBlsValidation notin flags: + var agg {.noInit.}: AggregateSignature + agg.init(tgt.signature) + agg.aggregate(src.signature) + tgt.signature = agg.finish() + template wrappedTimedTest(name: string, body: untyped) = # `check` macro takes a copy of whatever it's checking, on the stack! block: # Symbol namespacing diff --git a/tests/test_interop.nim b/tests/test_interop.nim index b48e325ab..21fe472a0 100644 --- a/tests/test_interop.nim +++ b/tests/test_interop.nim @@ -124,7 +124,7 @@ suiteReport "Interop": check: # getBytes is bigendian and returns full 48 bytes of key.. - Uint256.fromBytesBE(key.toRaw()[48-32..<48]) == v + Uint256.fromBytesBE(key.toRaw()) == v timedTest "Interop signatures": for dep in depositsConfig: diff --git a/tests/test_keystore.nim b/tests/test_keystore.nim index 532dbd13e..a6c021504 100644 --- a/tests/test_keystore.nim +++ b/tests/test_keystore.nim @@ -15,8 +15,13 @@ import from strutils import replace -template `==`*(a, b: ValidatorPrivKey): bool = - blscurve.SecretKey(a) == blscurve.SecretKey(b) +func isEqual*(a, b: ValidatorPrivKey): bool = + # `==` on secret keys is not allowed + let pa = cast[ptr UncheckedArray[byte]](a.unsafeAddr) + let pb = cast[ptr UncheckedArray[byte]](b.unsafeAddr) + result = true + for i in 0 ..< sizeof(a): + result = result and pa[i] == pb[i] const scryptVector = """{ @@ -103,7 +108,7 @@ suiteReport "Keystore": decrypt = decryptKeystore(keystore, KeystorePass password) check decrypt.isOk - check secret == decrypt.get() + check secret.isEqual(decrypt.get()) timedTest "Scrypt decryption": let @@ -111,7 +116,7 @@ suiteReport "Keystore": decrypt = decryptKeystore(keystore, KeystorePass password) check decrypt.isOk - check secret == decrypt.get() + check secret.isEqual(decrypt.get()) timedTest "Pbkdf2 encryption": let keystore = createKeystore(kdfPbkdf2, rng[], secret, diff --git a/tests/test_state_transition.nim b/tests/test_state_transition.nim index 0a81ff3cc..956a006f9 100644 --- a/tests/test_state_transition.nim +++ b/tests/test_state_transition.nim @@ -8,9 +8,9 @@ {.used.} import - unittest, + unittest, chronicles, ./testutil, ./testblockutil, - ../beacon_chain/spec/[beaconstate, datatypes, digest, + ../beacon_chain/spec/[beaconstate, datatypes, digest, crypto, validator, state_transition, presets], ../beacon_chain/ssz diff --git a/tests/testblockutil.nim b/tests/testblockutil.nim index b1a9f6d25..e9817c3a1 100644 --- a/tests/testblockutil.nim +++ b/tests/testblockutil.nim @@ -233,14 +233,18 @@ proc makeFullAttestations*( state.fork, state.genesis_validators_root, data, hackPrivKey(state.validators[committee[0]])) ) + var agg {.noInit.}: AggregateSignature + agg.init(attestation.signature) + # Aggregate the remainder attestation.aggregation_bits.setBit 0 for j in 1 ..< committee.len(): attestation.aggregation_bits.setBit j if skipBLSValidation notin flags: - attestation.signature.aggregate(get_attestation_signature( + agg.aggregate(get_attestation_signature( state.fork, state.genesis_validators_root, data, hackPrivKey(state.validators[committee[j]]) )) + attestation.signature = agg.finish() result.add attestation diff --git a/vendor/nim-blscurve b/vendor/nim-blscurve index 271a57385..2317e9cea 160000 --- a/vendor/nim-blscurve +++ b/vendor/nim-blscurve @@ -1 +1 @@ -Subproject commit 271a57385067556612dd38189ddac714b82edbf7 +Subproject commit 2317e9ceac3378f9540bbbbc0b1fdbaf73808a9b