Bump nim-blscurve (#1491)

* Bump BLSCurve

* Use unified aggregation API

* use new blscurve with unified aggregate API

* bump

* fix toRaw

* replace state_sim combine with AggregateSignature

* Fix 32-bit

* Fix 32-bit for real and test deactivating ccache for fno-tree-lopp-vectorize flag

* change compilation switches to narrow down Linux issue

* Use -fno-tree-vectorize to disable both tree-loop-vectorize and tree-slp-vectorize

* blscurve now disables both Loop and SLP vectorization

* Add tests for the miracl/milagro fallback

* Travis has max log size of 4MB

* Test with Miracl in the finalization test

* fix state_sim log level

* Coment out the slow fallback tests
This commit is contained in:
Mamy Ratsimbazafy 2020-08-15 19:33:58 +02:00 committed by GitHub
parent 37cb0a2249
commit 454b9d0724
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 121 additions and 78 deletions

View File

@ -53,4 +53,3 @@ script:
- make -j${NPROC} NIMFLAGS="--parallelBuild:2" LOG_LEVEL=TRACE
- make -j${NPROC} NIMFLAGS="--parallelBuild:2 -d:testnet_servers_image" LOG_LEVEL=TRACE beacon_node
- make -j${NPROC} NIMFLAGS="--parallelBuild:2" DISABLE_TEST_FIXTURES_SCRIPT=1 test

9
Jenkinsfile vendored
View File

@ -51,6 +51,14 @@ def runStages() {
./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
"""
}
// stage("testnet finalization - Miracl/Milagro fallback") {
// // EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node
// sh """#!/bin/bash
// set -e
// NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
// NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
// """
// }
}
)
}
@ -92,4 +100,3 @@ parallel(
}
}
)

View File

@ -59,21 +59,32 @@ task test, "Run all tests":
# price we pay for that.
# Just the part of minimal config which explicitly differs from mainnet
buildAndRunBinary "test_fixture_const_sanity_check", "tests/official/", "-d:const_preset=minimal"
buildAndRunBinary "test_fixture_const_sanity_check", "tests/official/", """-d:const_preset=minimal -d:chronicles_sinks="json[file]""""
# Mainnet config
buildAndRunBinary "proto_array", "beacon_chain/fork_choice/", "-d:const_preset=mainnet"
buildAndRunBinary "fork_choice", "beacon_chain/fork_choice/", "-d:const_preset=mainnet"
buildAndRunBinary "all_tests", "tests/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
buildAndRunBinary "proto_array", "beacon_chain/fork_choice/", """-d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
buildAndRunBinary "fork_choice", "beacon_chain/fork_choice/", """-d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
buildAndRunBinary "all_tests", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
# Check Miracl/Milagro fallback on select tests
buildAndRunBinary "test_interop", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
buildAndRunBinary "test_process_attestation", "tests/spec_block_processing/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
buildAndRunBinary "test_process_deposits", "tests/spec_block_processing/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
buildAndRunBinary "test_attestation_pool", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
buildAndRunBinary "test_block_pool", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
# Generic SSZ test, doesn't use consensus objects minimal/mainnet presets
buildAndRunBinary "test_fixture_ssz_generic_types", "tests/official/", "-d:chronicles_log_level=TRACE"
buildAndRunBinary "test_fixture_ssz_generic_types", "tests/official/", """-d:chronicles_log_level=TRACE -d:chronicles_sinks="json[file]""""
# Consensus object SSZ tests
buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
# 0.12.1
buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
# State and block sims; getting to 4th epoch triggers consensus checks
buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet", "--validators=3000 --slots=128"
buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet -d:chronicles_log_level=INFO", "--validators=3000 --slots=128"
# buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_log_level=INFO", "--validators=3000 --slots=128"
buildAndRunBinary "block_sim", "research/", "-d:const_preset=mainnet", "--validators=3000 --slots=128"
# buildAndRunBinary "block_sim", "research/", "-d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl", "--validators=3000 --slots=128"

View File

@ -283,6 +283,9 @@ proc getAttestationsForBlock*(pool: AttestationPool,
signature: a.validations[0].aggregate_signature
)
agg {.noInit.}: AggregateSignature
agg.init(a.validations[0].aggregate_signature)
# TODO what's going on here is that when producing a block, we need to
# include only such attestations that will not cause block validation
# to fail. How this interacts with voting and the acceptance of
@ -308,8 +311,9 @@ proc getAttestationsForBlock*(pool: AttestationPool,
# one new attestation in there
if not attestation.aggregation_bits.overlaps(v.aggregation_bits):
attestation.aggregation_bits.combine(v.aggregation_bits)
attestation.signature.aggregate(v.aggregate_signature)
agg.aggregate(v.aggregate_signature)
attestation.signature = agg.finish()
result.add(attestation)
if result.lenu64 >= MAX_ATTESTATIONS:

View File

@ -44,7 +44,7 @@ export results, json_serialization
const
RawSigSize* = 96
RawPubKeySize* = 48
RawPrivKeySize* = 48
# RawPrivKeySize* = 48 for Miracl / 32 for BLST
type
BlsValueType* = enum
@ -77,6 +77,8 @@ type
SomeSig* = TrustedSig | ValidatorSig
export AggregateSignature
func `==`*(a, b: BlsValue): bool =
if a.kind != b.kind: return false
if a.kind == Real:
@ -125,10 +127,20 @@ proc initPubKey*(pubkey: ValidatorPubKey): ValidatorPubKey =
return ValidatorPubKey()
key.get
func aggregate*(x: var ValidatorSig, other: ValidatorSig) =
## Aggregate 2 Validator Signatures
func init*(agg: var AggregateSignature, sig: ValidatorSig) {.inline.}=
## Initializes an aggregate signature context
## This assumes that the signature is valid
agg.init(sig.blsValue)
func aggregate*(agg: var AggregateSignature, sig: ValidatorSig) {.inline.}=
## Aggregate two Validator Signatures
## This assumes that they are real signatures
x.blsValue.aggregate(other.blsValue)
agg.aggregate(sig.blsValue)
func finish*(agg: AggregateSignature): ValidatorSig {.inline.}=
## Canonicalize an AggregateSignature into a signature
result.kind = Real
result.blsValue.finish(agg)
# https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/beacon-chain.md#bls-signatures
proc blsVerify*(
@ -218,9 +230,14 @@ func `$`*(x: BlsValue): string =
else:
"raw: " & x.blob.toHex()
func toRaw*(x: ValidatorPrivKey): array[RawPrivKeySize, byte] =
func toRaw*(x: ValidatorPrivKey): array[32, byte] =
# TODO: distinct type - see https://github.com/status-im/nim-blscurve/pull/67
SecretKey(x).exportRaw()
when BLS_BACKEND == BLST:
result = SecretKey(x).exportRaw()
else:
# Miracl exports to 384-bit arrays, but Curve order is 256-bit
let raw = SecretKey(x).exportRaw()
result[0..32-1] = raw.toOpenArray(48-32, 48-1)
func toRaw*(x: BlsValue): auto =
if x.kind == Real:

View File

@ -79,4 +79,3 @@ switch("warning", "LockLevel:off")
# Useful for Chronos metrics.
--define:chronosFutureTracking

View File

@ -99,19 +99,3 @@ proc printTimers*[Timers: enum](
echo "Validators: ", state.validators.len, ", epoch length: ", SLOTS_PER_EPOCH
echo "Validators per attestation (mean): ", attesters.mean
printTimers(validate, timers)
proc combine*(tgt: var Attestation, src: Attestation, flags: UpdateFlags) =
## Combine the signature and participation bitfield, with the assumption that
## the same data is being signed - if the signatures overlap, they are not
## combined.
doAssert tgt.data == src.data
# In a BLS aggregate signature, one needs to count how many times a
# particular public key has been added - since we use a single bit per key, we
# can only it once, thus we can never combine signatures that overlap already!
if not tgt.aggregation_bits.overlaps(src.aggregation_bits):
tgt.aggregation_bits.combine(src.aggregation_bits)
if skipBlsValidation notin flags:
tgt.signature.aggregate(src.signature)

View File

@ -124,18 +124,24 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
attesters.push scas.len()
withTimer(timers[tAttest]):
var agg {.noInit.}: AggregateSignature
for v in scas:
if (rand(r, high(int)).float * attesterRatio).int <= high(int):
if first:
attestation =
makeAttestation(state[].data, latest_block_root, scas, target_slot,
i.uint64, v, cache, flags)
agg.init(attestation.signature)
first = false
else:
attestation.combine(
let att2 =
makeAttestation(state[].data, latest_block_root, scas, target_slot,
i.uint64, v, cache, flags),
flags)
i.uint64, v, cache, flags)
if not att2.aggregation_bits.overlaps(attestation.aggregation_bits):
attestation.aggregation_bits.combine(att2.aggregation_bits)
if skipBlsValidation notin flags:
agg.aggregate(att2.signature)
attestation.signature = agg.finish()
if not first:
# add the attestation if any of the validators attested, as given

View File

@ -160,7 +160,7 @@ else
fi
NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK})
$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract
$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract
PIDS=""
WEB3_ARG=""
@ -347,4 +347,3 @@ else
exit 1
fi
fi

View File

@ -7,29 +7,10 @@
import
macros,
nimcrypto/utils,
../../beacon_chain/spec/[datatypes, crypto, digest], ../../beacon_chain/ssz/types
# digest is necessary for them to be printed as hex
# Define comparison of object variants for BLSValue
# https://github.com/nim-lang/Nim/issues/6676
# (fully generic available - see also https://github.com/status-im/nim-beacon-chain/commit/993789bad684721bd7c74ea14b35c2d24dbb6e51)
# ----------------------------------------------------------------
proc `==`*(a, b: BlsValue): bool =
## We sometimes need to compare real BlsValue
## from parsed opaque blobs that are not really on the BLS curve
## and full of zeros
if a.kind == Real:
if b.kind == Real:
a.blsvalue == b.blsValue
else:
$a.blsvalue == toHex(b.blob, true)
else:
if b.kind == Real:
toHex(a.blob, true) == $b.blsValue
else:
a.blob == b.blob
export crypto.`==`
# ---------------------------------------------------------------------
@ -48,9 +29,10 @@ proc compareStmt(xSubField, ySubField: NimNode, stmts: var NimNode) =
let xStr = $xSubField.toStrLit
let yStr = $ySubField.toStrLit
let isEqual = bindSym("==") # Bind all expose equality, in particular for BlsValue
stmts.add quote do:
doAssert(
`xSubField` == `ySubField`,
`isEqual`(`xSubField`, `ySubField`),
"\nDiff: " & `xStr` & " = " & $`xSubField` & "\n" &
"and " & `yStr` & " = " & $`ySubField` & "\n"
)
@ -59,16 +41,16 @@ proc compareContainerStmt(xSubField, ySubField: NimNode, stmts: var NimNode) =
let xStr = $xSubField.toStrLit
let yStr = $ySubField.toStrLit
let isEqual = bindSym("==") # Bind all expose equality, in particular for BlsValue
stmts.add quote do:
doAssert(
`xSubField`.len == `ySubField`.len,
`isEqual`(`xSubField`.len, `ySubField`.len),
"\nDiff: " & `xStr` & ".len = " & $`xSubField`.len & "\n" &
"and " & `yStr` & ".len = " & $`ySubField`.len & "\n"
)
for idx in `xSubField`.low .. `xSubField`.high:
doAssert(
`xSubField`[idx] == `ySubField`[idx],
`isEqual`(`xSubField`[idx], `ySubField`[idx]),
"\nDiff: " & `xStr` & "[" & $idx & "] = " & $`xSubField`[idx] & "\n" &
"and " & `yStr` & "[" & $idx & "] = " & $`ySubField`[idx] & "\n"
)

View File

@ -11,6 +11,8 @@
import
# Standard library
sets,
# Status
chronicles,
# Specs
../../beacon_chain/spec/[datatypes, beaconstate, helpers, validator, crypto,
signatures, state_transition, presets],
@ -63,6 +65,7 @@ proc signMockAttestation*(state: BeaconState, attestation: var Attestation) =
cache
)
var agg {.noInit.}: AggregateSignature
var first_iter = true # Can't do while loop on hashset
for validator_index in participants:
let sig = get_attestation_signature(
@ -70,10 +73,14 @@ proc signMockAttestation*(state: BeaconState, attestation: var Attestation) =
MockPrivKeys[validator_index]
)
if first_iter:
attestation.signature = sig
agg.init(sig)
first_iter = false
else:
aggregate(attestation.signature, sig)
agg.aggregate(sig)
if first_iter != true:
attestation.signature = agg.finish()
# Otherwise no participants so zero sig
proc mockAttestationImpl(
state: BeaconState,

View File

@ -10,7 +10,7 @@
import
bearssl, eth/keys,
blscurve/bls_signature_scheme,
blscurve,
../../beacon_chain/spec/[datatypes, crypto, presets]
proc newKeyPair(rng: var BrHmacDrbgContext): BlsResult[tuple[pub: ValidatorPubKey, priv: ValidatorPrivKey]] =
@ -25,7 +25,7 @@ proc newKeyPair(rng: var BrHmacDrbgContext): BlsResult[tuple[pub: ValidatorPubKe
var
sk: SecretKey
pk: bls_signature_scheme.PublicKey
pk: blscurve.PublicKey
if keyGen(ikm, pk, sk):
ok((ValidatorPubKey(kind: Real, blsValue: pk), ValidatorPrivKey(sk)))
else:

View File

@ -13,7 +13,7 @@ import
# Utilities
stew/results,
# Beacon chain internals
../../beacon_chain/spec/[datatypes, state_transition_block],
../../beacon_chain/spec/[datatypes, state_transition_block, crypto],
../../beacon_chain/ssz,
# Test utilities
../testutil,

View File

@ -11,13 +11,32 @@ import
unittest,
chronicles,
stew/byteutils,
./testutil, ./testblockutil, ../research/simutils,
./testutil, ./testblockutil,
../beacon_chain/spec/[crypto, datatypes, digest, validator, state_transition,
helpers, beaconstate, presets],
../beacon_chain/[beacon_node_types, attestation_pool, extras],
../beacon_chain/fork_choice/[fork_choice_types, fork_choice],
../beacon_chain/block_pools/[chain_dag, clearance]
func combine(tgt: var Attestation, src: Attestation, flags: UpdateFlags) =
## Combine the signature and participation bitfield, with the assumption that
## the same data is being signed - if the signatures overlap, they are not
## combined.
doAssert tgt.data == src.data
# In a BLS aggregate signature, one needs to count how many times a
# particular public key has been added - since we use a single bit per key, we
# can only it once, thus we can never combine signatures that overlap already!
if not tgt.aggregation_bits.overlaps(src.aggregation_bits):
tgt.aggregation_bits.combine(src.aggregation_bits)
if skipBlsValidation notin flags:
var agg {.noInit.}: AggregateSignature
agg.init(tgt.signature)
agg.aggregate(src.signature)
tgt.signature = agg.finish()
template wrappedTimedTest(name: string, body: untyped) =
# `check` macro takes a copy of whatever it's checking, on the stack!
block: # Symbol namespacing

View File

@ -124,7 +124,7 @@ suiteReport "Interop":
check:
# getBytes is bigendian and returns full 48 bytes of key..
Uint256.fromBytesBE(key.toRaw()[48-32..<48]) == v
Uint256.fromBytesBE(key.toRaw()) == v
timedTest "Interop signatures":
for dep in depositsConfig:

View File

@ -15,8 +15,13 @@ import
from strutils import replace
template `==`*(a, b: ValidatorPrivKey): bool =
blscurve.SecretKey(a) == blscurve.SecretKey(b)
func isEqual*(a, b: ValidatorPrivKey): bool =
# `==` on secret keys is not allowed
let pa = cast[ptr UncheckedArray[byte]](a.unsafeAddr)
let pb = cast[ptr UncheckedArray[byte]](b.unsafeAddr)
result = true
for i in 0 ..< sizeof(a):
result = result and pa[i] == pb[i]
const
scryptVector = """{
@ -103,7 +108,7 @@ suiteReport "Keystore":
decrypt = decryptKeystore(keystore, KeystorePass password)
check decrypt.isOk
check secret == decrypt.get()
check secret.isEqual(decrypt.get())
timedTest "Scrypt decryption":
let
@ -111,7 +116,7 @@ suiteReport "Keystore":
decrypt = decryptKeystore(keystore, KeystorePass password)
check decrypt.isOk
check secret == decrypt.get()
check secret.isEqual(decrypt.get())
timedTest "Pbkdf2 encryption":
let keystore = createKeystore(kdfPbkdf2, rng[], secret,

View File

@ -8,9 +8,9 @@
{.used.}
import
unittest,
unittest, chronicles,
./testutil, ./testblockutil,
../beacon_chain/spec/[beaconstate, datatypes, digest,
../beacon_chain/spec/[beaconstate, datatypes, digest, crypto,
validator, state_transition, presets],
../beacon_chain/ssz

View File

@ -233,14 +233,18 @@ proc makeFullAttestations*(
state.fork, state.genesis_validators_root, data,
hackPrivKey(state.validators[committee[0]]))
)
var agg {.noInit.}: AggregateSignature
agg.init(attestation.signature)
# Aggregate the remainder
attestation.aggregation_bits.setBit 0
for j in 1 ..< committee.len():
attestation.aggregation_bits.setBit j
if skipBLSValidation notin flags:
attestation.signature.aggregate(get_attestation_signature(
agg.aggregate(get_attestation_signature(
state.fork, state.genesis_validators_root, data,
hackPrivKey(state.validators[committee[j]])
))
attestation.signature = agg.finish()
result.add attestation

2
vendor/nim-blscurve vendored

@ -1 +1 @@
Subproject commit 271a57385067556612dd38189ddac714b82edbf7
Subproject commit 2317e9ceac3378f9540bbbbc0b1fdbaf73808a9b