Bump nim-blscurve (#1491)

* Bump BLSCurve

* Use unified aggregation API

* use new blscurve with unified aggregate API

* bump

* fix toRaw

* replace state_sim combine with AggregateSignature

* Fix 32-bit

* Fix 32-bit for real and test deactivating ccache for fno-tree-lopp-vectorize flag

* change compilation switches to narrow down Linux issue

* Use -fno-tree-vectorize to disable both tree-loop-vectorize and tree-slp-vectorize

* blscurve now disables both Loop and SLP vectorization

* Add tests for the miracl/milagro fallback

* Travis has max log size of 4MB

* Test with Miracl in the finalization test

* fix state_sim log level

* Coment out the slow fallback tests

.travis.yml

@@ -53,4 +53,3 @@ script:
   - make -j${NPROC} NIMFLAGS="--parallelBuild:2" LOG_LEVEL=TRACE
   - make -j${NPROC} NIMFLAGS="--parallelBuild:2 -d:testnet_servers_image" LOG_LEVEL=TRACE beacon_node
   - make -j${NPROC} NIMFLAGS="--parallelBuild:2" DISABLE_TEST_FIXTURES_SCRIPT=1 test
-

Jenkinsfile

@@ -51,6 +51,14 @@ def runStages() {
 						./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
 						"""
 					}
+					// stage("testnet finalization - Miracl/Milagro fallback") {
+					// 	// EXECUTOR_NUMBER will be 0 or 1, since we have 2 executors per Jenkins node
+					// 	sh """#!/bin/bash
+					// 	set -e
+					// 	NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 0 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet0_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
+					// 	NIMFLAGS="-d:BLS_FORCE_BACKEND=miracl" ./scripts/launch_local_testnet.sh --testnet 1 --nodes 4 --log-level INFO --disable-htop --data-dir local_testnet1_data --base-port \$(( 9000 + EXECUTOR_NUMBER * 100 )) --base-metrics-port \$(( 8008 + EXECUTOR_NUMBER * 100 )) -- --verify-finalization --stop-at-epoch=5
+					// 	"""
+					// }
 				}
 			)
 		}
@@ -92,4 +100,3 @@ parallel(
 		}
 	}
 )
-

beacon_chain.nimble

@@ -59,21 +59,32 @@ task test, "Run all tests":
   # price we pay for that.
 
   # Just the part of minimal config which explicitly differs from mainnet
-  buildAndRunBinary "test_fixture_const_sanity_check", "tests/official/", "-d:const_preset=minimal"
+  buildAndRunBinary "test_fixture_const_sanity_check", "tests/official/", """-d:const_preset=minimal -d:chronicles_sinks="json[file]""""
 
   # Mainnet config
-  buildAndRunBinary "proto_array", "beacon_chain/fork_choice/", "-d:const_preset=mainnet"
+  buildAndRunBinary "proto_array", "beacon_chain/fork_choice/", """-d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
-  buildAndRunBinary "fork_choice", "beacon_chain/fork_choice/", "-d:const_preset=mainnet"
+  buildAndRunBinary "fork_choice", "beacon_chain/fork_choice/", """-d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
-  buildAndRunBinary "all_tests", "tests/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
+  buildAndRunBinary "all_tests", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
+
+  # Check Miracl/Milagro fallback on select tests
+  buildAndRunBinary "test_interop", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
+  buildAndRunBinary "test_process_attestation", "tests/spec_block_processing/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
+  buildAndRunBinary "test_process_deposits", "tests/spec_block_processing/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
+  buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
+  buildAndRunBinary "test_attestation_pool", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
+  buildAndRunBinary "test_block_pool", "tests/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_sinks="json[file]""""
 
   # Generic SSZ test, doesn't use consensus objects minimal/mainnet presets
-  buildAndRunBinary "test_fixture_ssz_generic_types", "tests/official/", "-d:chronicles_log_level=TRACE"
+  buildAndRunBinary "test_fixture_ssz_generic_types", "tests/official/", """-d:chronicles_log_level=TRACE -d:chronicles_sinks="json[file]""""
 
   # Consensus object SSZ tests
-  buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
+  buildAndRunBinary "test_fixture_ssz_consensus_objects", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
 
-  buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", "-d:chronicles_log_level=TRACE -d:const_preset=mainnet"
+  # 0.12.1
+  buildAndRunBinary "all_fixtures_require_ssz", "tests/official/", """-d:chronicles_log_level=TRACE -d:const_preset=mainnet -d:chronicles_sinks="json[file]""""
 
   # State and block sims; getting to 4th epoch triggers consensus checks
-  buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet", "--validators=3000 --slots=128"
+  buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet -d:chronicles_log_level=INFO", "--validators=3000 --slots=128"
+  # buildAndRunBinary "state_sim", "research/", "-d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl -d:chronicles_log_level=INFO", "--validators=3000 --slots=128"
   buildAndRunBinary "block_sim", "research/", "-d:const_preset=mainnet", "--validators=3000 --slots=128"
+  # buildAndRunBinary "block_sim", "research/", "-d:const_preset=mainnet -d:BLS_FORCE_BACKEND=miracl", "--validators=3000 --slots=128"

beacon_chain/attestation_pool.nim

@@ -283,6 +283,9 @@ proc getAttestationsForBlock*(pool: AttestationPool,
         signature: a.validations[0].aggregate_signature
       )
 
+      agg {.noInit.}: AggregateSignature
+    agg.init(a.validations[0].aggregate_signature)
+
     # TODO what's going on here is that when producing a block, we need to
     #      include only such attestations that will not cause block validation
     #      to fail. How this interacts with voting and the acceptance of
@@ -308,8 +311,9 @@ proc getAttestationsForBlock*(pool: AttestationPool,
       #      one new attestation in there
       if not attestation.aggregation_bits.overlaps(v.aggregation_bits):
         attestation.aggregation_bits.combine(v.aggregation_bits)
-        attestation.signature.aggregate(v.aggregate_signature)
+        agg.aggregate(v.aggregate_signature)
 
+    attestation.signature = agg.finish()
     result.add(attestation)
 
     if result.lenu64 >= MAX_ATTESTATIONS:

beacon_chain/spec/crypto.nim

@@ -44,7 +44,7 @@ export results, json_serialization
 const
   RawSigSize* = 96
   RawPubKeySize* = 48
-  RawPrivKeySize* = 48
+  # RawPrivKeySize* = 48 for Miracl / 32 for BLST
 
 type
   BlsValueType* = enum
@@ -77,6 +77,8 @@ type
 
   SomeSig* = TrustedSig | ValidatorSig
 
+export AggregateSignature
+
 func `==`*(a, b: BlsValue): bool =
   if a.kind != b.kind: return false
   if a.kind == Real:
@@ -125,10 +127,20 @@ proc initPubKey*(pubkey: ValidatorPubKey): ValidatorPubKey =
     return ValidatorPubKey()
   key.get
 
-func aggregate*(x: var ValidatorSig, other: ValidatorSig) =
+func init*(agg: var AggregateSignature, sig: ValidatorSig) {.inline.}=
-  ## Aggregate 2 Validator Signatures
+  ## Initializes an aggregate signature context
+  ## This assumes that the signature is valid
+  agg.init(sig.blsValue)
+
+func aggregate*(agg: var AggregateSignature, sig: ValidatorSig) {.inline.}=
+  ## Aggregate two Validator Signatures
   ## This assumes that they are real signatures
-  x.blsValue.aggregate(other.blsValue)
+  agg.aggregate(sig.blsValue)
+
+func finish*(agg: AggregateSignature): ValidatorSig {.inline.}=
+  ## Canonicalize an AggregateSignature into a signature
+  result.kind = Real
+  result.blsValue.finish(agg)
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.2/specs/phase0/beacon-chain.md#bls-signatures
 proc blsVerify*(
@@ -218,9 +230,14 @@ func `$`*(x: BlsValue): string =
   else:
     "raw: " & x.blob.toHex()
 
-func toRaw*(x: ValidatorPrivKey): array[RawPrivKeySize, byte] =
+func toRaw*(x: ValidatorPrivKey): array[32, byte] =
   # TODO: distinct type - see https://github.com/status-im/nim-blscurve/pull/67
-  SecretKey(x).exportRaw()
+  when BLS_BACKEND == BLST:
+    result = SecretKey(x).exportRaw()
+  else:
+    # Miracl exports to 384-bit arrays, but Curve order is 256-bit
+    let raw = SecretKey(x).exportRaw()
+    result[0..32-1] = raw.toOpenArray(48-32, 48-1)
 
 func toRaw*(x: BlsValue): auto =
   if x.kind == Real:

config.nims

@@ -79,4 +79,3 @@ switch("warning", "LockLevel:off")
 
 # Useful for Chronos metrics.
 --define:chronosFutureTracking
-

research/simutils.nim

@@ -99,19 +99,3 @@ proc printTimers*[Timers: enum](
   echo "Validators: ", state.validators.len, ", epoch length: ", SLOTS_PER_EPOCH
   echo "Validators per attestation (mean): ", attesters.mean
   printTimers(validate, timers)
-
-proc combine*(tgt: var Attestation, src: Attestation, flags: UpdateFlags) =
-  ## Combine the signature and participation bitfield, with the assumption that
-  ## the same data is being signed - if the signatures overlap, they are not
-  ## combined.
-
-  doAssert tgt.data == src.data
-
-  # In a BLS aggregate signature, one needs to count how many times a
-  # particular public key has been added - since we use a single bit per key, we
-  # can only it once, thus we can never combine signatures that overlap already!
-  if not tgt.aggregation_bits.overlaps(src.aggregation_bits):
-    tgt.aggregation_bits.combine(src.aggregation_bits)
-
-    if skipBlsValidation notin flags:
-      tgt.signature.aggregate(src.signature)

research/state_sim.nim

@@ -124,18 +124,24 @@ cli do(slots = SLOTS_PER_EPOCH * 6,
         attesters.push scas.len()
 
         withTimer(timers[tAttest]):
+          var agg {.noInit.}: AggregateSignature
           for v in scas:
             if (rand(r, high(int)).float * attesterRatio).int <= high(int):
               if first:
                 attestation =
                   makeAttestation(state[].data, latest_block_root, scas, target_slot,
                     i.uint64, v, cache, flags)
+                agg.init(attestation.signature)
                 first = false
               else:
-                attestation.combine(
+                let att2 =
                   makeAttestation(state[].data, latest_block_root, scas, target_slot,
-                    i.uint64, v, cache, flags),
+                    i.uint64, v, cache, flags)
-                  flags)
+                if not att2.aggregation_bits.overlaps(attestation.aggregation_bits):
+                  attestation.aggregation_bits.combine(att2.aggregation_bits)
+                  if skipBlsValidation notin flags:
+                    agg.aggregate(att2.signature)
+          attestation.signature = agg.finish()
 
         if not first:
           # add the attestation if any of the validators attested, as given

scripts/launch_local_testnet.sh

@@ -160,7 +160,7 @@ else
 fi
 
 NETWORK_NIM_FLAGS=$(scripts/load-testnet-nim-flags.sh ${NETWORK})
-$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="-d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract
+$MAKE -j2 LOG_LEVEL="${LOG_LEVEL}" NIMFLAGS="${NIMFLAGS} -d:insecure -d:testnet_servers_image -d:local_testnet ${NETWORK_NIM_FLAGS}" beacon_node deposit_contract
 
 PIDS=""
 WEB3_ARG=""
@@ -347,4 +347,3 @@ else
     exit 1
   fi
 fi
-

tests/helpers/debug_state.nim

@@ -7,29 +7,10 @@
 
 import
   macros,
-  nimcrypto/utils,
   ../../beacon_chain/spec/[datatypes, crypto, digest], ../../beacon_chain/ssz/types
   # digest is necessary for them to be printed as hex
 
-# Define comparison of object variants for BLSValue
+export crypto.`==`
-# https://github.com/nim-lang/Nim/issues/6676
-# (fully generic available - see also https://github.com/status-im/nim-beacon-chain/commit/993789bad684721bd7c74ea14b35c2d24dbb6e51)
-# ----------------------------------------------------------------
-
-proc `==`*(a, b: BlsValue): bool =
-  ## We sometimes need to compare real BlsValue
-  ## from parsed opaque blobs that are not really on the BLS curve
-  ## and full of zeros
-  if a.kind == Real:
-    if b.kind == Real:
-      a.blsvalue == b.blsValue
-    else:
-      $a.blsvalue == toHex(b.blob, true)
-  else:
-    if b.kind == Real:
-      toHex(a.blob, true) == $b.blsValue
-    else:
-      a.blob == b.blob
 
 # ---------------------------------------------------------------------
 
@@ -48,9 +29,10 @@ proc compareStmt(xSubField, ySubField: NimNode, stmts: var NimNode) =
   let xStr = $xSubField.toStrLit
   let yStr = $ySubField.toStrLit
 
+  let isEqual = bindSym("==") # Bind all expose equality, in particular for BlsValue
   stmts.add quote do:
     doAssert(
-      `xSubField` == `ySubField`,
+      `isEqual`(`xSubField`, `ySubField`),
       "\nDiff: " & `xStr` & " = " & $`xSubField` & "\n" &
       "and   " & `yStr` & " = " & $`ySubField` & "\n"
     )
@@ -59,16 +41,16 @@ proc compareContainerStmt(xSubField, ySubField: NimNode, stmts: var NimNode) =
   let xStr = $xSubField.toStrLit
   let yStr = $ySubField.toStrLit
 
-
+  let isEqual = bindSym("==") # Bind all expose equality, in particular for BlsValue
   stmts.add quote do:
     doAssert(
-      `xSubField`.len == `ySubField`.len,
+      `isEqual`(`xSubField`.len, `ySubField`.len),
         "\nDiff: " & `xStr` & ".len = " & $`xSubField`.len & "\n" &
         "and   " & `yStr` & ".len = " & $`ySubField`.len & "\n"
     )
     for idx in `xSubField`.low .. `xSubField`.high:
       doAssert(
-        `xSubField`[idx] == `ySubField`[idx],
+        `isEqual`(`xSubField`[idx], `ySubField`[idx]),
         "\nDiff: " & `xStr` & "[" & $idx & "] = " & $`xSubField`[idx] & "\n" &
         "and   " & `yStr` & "[" & $idx & "] = " & $`ySubField`[idx] & "\n"
       )

tests/mocking/mock_attestations.nim

@@ -11,6 +11,8 @@
 import
   # Standard library
   sets,
+  # Status
+  chronicles,
   # Specs
   ../../beacon_chain/spec/[datatypes, beaconstate, helpers, validator, crypto,
                            signatures, state_transition, presets],
@@ -63,6 +65,7 @@ proc signMockAttestation*(state: BeaconState, attestation: var Attestation) =
     cache
   )
 
+  var agg {.noInit.}: AggregateSignature
   var first_iter = true # Can't do while loop on hashset
   for validator_index in participants:
     let sig = get_attestation_signature(
@@ -70,10 +73,14 @@ proc signMockAttestation*(state: BeaconState, attestation: var Attestation) =
       MockPrivKeys[validator_index]
     )
     if first_iter:
-      attestation.signature = sig
+      agg.init(sig)
       first_iter = false
     else:
-      aggregate(attestation.signature, sig)
+      agg.aggregate(sig)
+
+  if first_iter != true:
+    attestation.signature = agg.finish()
+    # Otherwise no participants so zero sig
 
 proc mockAttestationImpl(
        state: BeaconState,

tests/mocking/mock_validator_keys.nim

@@ -10,7 +10,7 @@
 
 import
   bearssl, eth/keys,
-  blscurve/bls_signature_scheme,
+  blscurve,
   ../../beacon_chain/spec/[datatypes, crypto, presets]
 
 proc newKeyPair(rng: var BrHmacDrbgContext): BlsResult[tuple[pub: ValidatorPubKey, priv: ValidatorPrivKey]] =
@@ -25,7 +25,7 @@ proc newKeyPair(rng: var BrHmacDrbgContext): BlsResult[tuple[pub: ValidatorPubKe
 
   var
     sk: SecretKey
-    pk: bls_signature_scheme.PublicKey
+    pk: blscurve.PublicKey
   if keyGen(ikm, pk, sk):
     ok((ValidatorPubKey(kind: Real, blsValue: pk), ValidatorPrivKey(sk)))
   else:

tests/official/test_fixture_operations_block_header.nim

@@ -13,7 +13,7 @@ import
   # Utilities
   stew/results,
   # Beacon chain internals
-  ../../beacon_chain/spec/[datatypes, state_transition_block],
+  ../../beacon_chain/spec/[datatypes, state_transition_block, crypto],
   ../../beacon_chain/ssz,
   # Test utilities
   ../testutil,

tests/test_attestation_pool.nim

@@ -11,13 +11,32 @@ import
   unittest,
   chronicles,
   stew/byteutils,
-  ./testutil, ./testblockutil, ../research/simutils,
+  ./testutil, ./testblockutil,
   ../beacon_chain/spec/[crypto, datatypes, digest, validator, state_transition,
                         helpers, beaconstate, presets],
   ../beacon_chain/[beacon_node_types, attestation_pool, extras],
   ../beacon_chain/fork_choice/[fork_choice_types, fork_choice],
   ../beacon_chain/block_pools/[chain_dag, clearance]
 
+func combine(tgt: var Attestation, src: Attestation, flags: UpdateFlags) =
+  ## Combine the signature and participation bitfield, with the assumption that
+  ## the same data is being signed - if the signatures overlap, they are not
+  ## combined.
+
+  doAssert tgt.data == src.data
+
+  # In a BLS aggregate signature, one needs to count how many times a
+  # particular public key has been added - since we use a single bit per key, we
+  # can only it once, thus we can never combine signatures that overlap already!
+  if not tgt.aggregation_bits.overlaps(src.aggregation_bits):
+    tgt.aggregation_bits.combine(src.aggregation_bits)
+
+    if skipBlsValidation notin flags:
+      var agg {.noInit.}: AggregateSignature
+      agg.init(tgt.signature)
+      agg.aggregate(src.signature)
+      tgt.signature = agg.finish()
+
 template wrappedTimedTest(name: string, body: untyped) =
   # `check` macro takes a copy of whatever it's checking, on the stack!
   block: # Symbol namespacing

tests/test_interop.nim

@@ -124,7 +124,7 @@ suiteReport "Interop":
 
       check:
         # getBytes is bigendian and returns full 48 bytes of key..
-        Uint256.fromBytesBE(key.toRaw()[48-32..<48]) == v
+        Uint256.fromBytesBE(key.toRaw()) == v
 
   timedTest "Interop signatures":
     for dep in depositsConfig:

tests/test_keystore.nim

@@ -15,8 +15,13 @@ import
 
 from strutils import replace
 
-template `==`*(a, b: ValidatorPrivKey): bool =
+func isEqual*(a, b: ValidatorPrivKey): bool =
-  blscurve.SecretKey(a) == blscurve.SecretKey(b)
+  # `==` on secret keys is not allowed
+  let pa = cast[ptr UncheckedArray[byte]](a.unsafeAddr)
+  let pb = cast[ptr UncheckedArray[byte]](b.unsafeAddr)
+  result = true
+  for i in 0 ..< sizeof(a):
+    result = result and pa[i] == pb[i]
 
 const
   scryptVector = """{
@@ -103,7 +108,7 @@ suiteReport "Keystore":
       decrypt = decryptKeystore(keystore, KeystorePass password)
 
     check decrypt.isOk
-    check secret == decrypt.get()
+    check secret.isEqual(decrypt.get())
 
   timedTest "Scrypt decryption":
     let
@@ -111,7 +116,7 @@ suiteReport "Keystore":
       decrypt = decryptKeystore(keystore, KeystorePass password)
 
     check decrypt.isOk
-    check secret == decrypt.get()
+    check secret.isEqual(decrypt.get())
 
   timedTest "Pbkdf2 encryption":
     let keystore = createKeystore(kdfPbkdf2, rng[], secret,

tests/test_state_transition.nim

@@ -8,9 +8,9 @@
 {.used.}
 
 import
-  unittest,
+  unittest, chronicles,
   ./testutil, ./testblockutil,
-  ../beacon_chain/spec/[beaconstate, datatypes, digest,
+  ../beacon_chain/spec/[beaconstate, datatypes, digest, crypto,
                         validator, state_transition, presets],
   ../beacon_chain/ssz
 

tests/testblockutil.nim

@@ -233,14 +233,18 @@ proc makeFullAttestations*(
         state.fork, state.genesis_validators_root, data,
         hackPrivKey(state.validators[committee[0]]))
     )
+    var agg {.noInit.}: AggregateSignature
+    agg.init(attestation.signature)
+
     # Aggregate the remainder
     attestation.aggregation_bits.setBit 0
     for j in 1 ..< committee.len():
       attestation.aggregation_bits.setBit j
       if skipBLSValidation notin flags:
-        attestation.signature.aggregate(get_attestation_signature(
+        agg.aggregate(get_attestation_signature(
           state.fork, state.genesis_validators_root, data,
           hackPrivKey(state.validators[committee[j]])
         ))
 
+    attestation.signature = agg.finish()
     result.add attestation

vendor/nim-blscurve

@@ -1 +1 @@
-Subproject commit 271a57385067556612dd38189ddac714b82edbf7
+Subproject commit 2317e9ceac3378f9540bbbbc0b1fdbaf73808a9b