nimbus-eth2/beacon_chain/spec/digest.nim

111 lines
3.4 KiB
Nim
Raw Normal View History

# beacon_chain
# Copyright (c) 2018-2020 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# Serenity hash function / digest
#
# https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/specs/phase0/beacon-chain.md#hash
#
# In Phase 0 the beacon chain is deployed with SHA256 (SHA2-256).
# Note that is is different from Keccak256 (often mistakenly called SHA3-256)
# and SHA3-256.
2019-03-09 20:34:08 +00:00
#
# In Eth1.0, the default hash function is Keccak256 and SHA256 is available as a precompiled contract.
2019-03-09 20:34:08 +00:00
#
# In our code base, to enable a smooth transition
# (already did Blake2b --> Keccak256 --> SHA2-256),
# we call this function `eth2digest`, and it outputs a `Eth2Digest`. Easy to sed :)
{.push raises: [Defect].}
import
# Standard library
std/hashes,
#Status libraries
chronicles,
nimcrypto/[sha2, hash],
stew/byteutils,
eth/common/eth_types_json_serialization,
blscurve
export
hash.`$`, sha2, readValue, writeValue
type
Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
when BLS_BACKEND == BLST:
export blscurve.update
type Eth2DigestCtx* = BLST_SHA256_CTX
else:
type Eth2DigestCtx* = sha2.sha256
func shortLog*(x: Eth2Digest): string =
x.data.toOpenArray(0, 3).toHex()
chronicles.formatIt Eth2Digest:
shortLog(it)
# TODO: expose an in-place digest function
# when hashing in loop or into a buffer
# See: https://github.com/cheatfate/nimcrypto/blob/b90ba3abd/nimcrypto/sha2.nim#L570
func eth2digest*(v: openArray[byte]): Eth2Digest {.noInit.} =
## Apply the Eth2 Hash function
## Do NOT use for secret data.
when BLS_BACKEND == BLST:
# BLST has a fast assembly optimized SHA256
result.data.bls_sha256_digest(v)
else:
# We use the init-update-finish interface to avoid
# the expensive burning/clearing memory (20~30% perf)
var ctx: Eth2DigestCtx
ctx.init()
ctx.update(v)
ctx.finish()
when BLS_BACKEND == BLST:
func update*(ctx: var BLST_SHA256_CTX; digest: Eth2Digest) =
ctx.update digest.data
func update*(ctx: var sha256; digest: Eth2Digest) =
v0.8.1 tests refactor (#326) * Introduce new mocking proc to replace: - makeFakeValidatorPrivKey - hackPrivKey - getNextBeaconProposerIndex - addBlock - makeBlock * Add comments on datastructure unsynced with the spec * Add merkle tree constructor and initial mocking for deposits (missing merkle proofs) * [Mock] Implement sparse merkle tree and merkle proof builder * [Mocking] Genesis deposits * Add compact_committees_roots init + mock genesis state * [Tests] Add first deposit test using the new mocking procedures * [Tests -deposits] add at and over 32 ETH deposit tests * [Tests - deposits] Add test for validator top-up * [Tests -deposits] Mention the TODO to test for invalid conditions * [Tests] Add stub to test "is_valid_genesis_state" * [Merkle proofs] Implement round-trip checks * Deactivate roundtrips test * SSZ - use EF convention for hash_tree_root / hashTreeRoot * [Tests - Attestation] Attestation mocking + initial test * Add mocking + 3 new tests for valid attestations + mention future invalid attestation tests * Add crosslinks test (1 failing to attestations in block being duplicated in state transition) * Single attestation crosslink test - workaround https://github.com/status-im/nim-beacon-chain/issues/361 * Add test for failed crosslink penalty * Rebase fixes + add refactored tests to test suite * justif-finalization helpers first batch * Add 234 finalization tests * Fix justif test, Rule I 234 finalization does not happen with sufficient support. (Also unittest check template does not fail properly in some cases) * Add tests for all finalization rules * Properly delete nim-byteutils following https://github.com/status-im/nim-beacon-chain/commit/c91727e7e5f9c7a95a634055e94e9d3bcb6cc41a#diff-7c3613dba5171cb6027c67835dd3b9d4 * use digest helper for deposit root
2019-08-28 12:07:00 +00:00
ctx.update digest.data
template withEth2Hash*(body: untyped): Eth2Digest =
## This little helper will init the hash function and return the sliced
## hash:
## let hashOfData = withHash: h.update(data)
when nimvm:
# In SSZ, computeZeroHashes require compile-time SHA256
block:
var h {.inject.}: sha256
init(h)
body
finish(h)
else:
when BLS_BACKEND == BLST:
block:
var h {.inject, noInit.}: Eth2DigestCtx
init(h)
body
var res {.noInit.}: Eth2Digest
finalize(res.data, h)
res
else:
block:
var h {.inject, noInit.}: Eth2DigestCtx
init(h)
body
finish(h)
2019-01-08 17:28:21 +00:00
performance fixes (#2259) * performance fixes * don't mark tree cache as dirty on read-only List accesses * store only blob in memory for keys and signatures, parse blob lazily * compare public keys by blob instead of parsing / converting to raw * compare Eth2Digest using non-constant-time comparison * avoid some unnecessary validator copying This branch will in particular speed up deposit processing which has been slowing down block replay. Pre (mainnet, 1600 blocks): ``` All time are ms Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB 0.417, 0.822, 0.036, 21.098, 1400, Load block from database 16.521, 0.000, 16.521, 16.521, 1, Load state from database 27.906, 50.846, 8.104, 1507.633, 1350, Apply block 52.617, 37.029, 20.640, 135.938, 50, Apply epoch block ``` Post: ``` 3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB 0.080, 0.560, 0.035, 21.015, 1400, Load block from database 17.595, 0.000, 17.595, 17.595, 1, Load state from database 15.706, 11.028, 8.300, 107.537, 1350, Apply block 33.217, 12.622, 17.331, 60.580, 50, Apply epoch block ``` * more perf fixes * load EpochRef cache into StateCache more aggressively * point out security concern with public key cache * reuse proposer index from state when processing block * avoid genericAssign in a few more places * don't parse key when signature is unparseable * fix `==` overload for Eth2Digest * preallocate validator list when getting active validators * speed up proposer index calculation a little bit * reuse cache when replaying blocks in ncli_db * avoid a few more copying loops ``` Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB 0.072, 0.357, 0.035, 13.400, 1400, Load block from database 17.295, 0.000, 17.295, 17.295, 1, Load state from database 5.918, 9.896, 0.198, 98.028, 1350, Apply block 15.888, 10.951, 7.902, 39.535, 50, Apply epoch block 0.000, 0.000, 0.000, 0.000, 0, Database block store ``` * clear full balance cache before processing rewards and penalties ``` All time are ms Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB 0.124, 0.506, 0.026, 202.370, 363345, Load block from database 97.614, 0.000, 97.614, 97.614, 1, Load state from database 0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch 14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch 1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing 0.000, 0.000, 0.000, 0.000, 0, Database block store ```
2021-01-25 12:04:18 +00:00
template hash*(x: Eth2Digest): Hash =
## Hash for digests for Nim hash tables
performance fixes (#2259) * performance fixes * don't mark tree cache as dirty on read-only List accesses * store only blob in memory for keys and signatures, parse blob lazily * compare public keys by blob instead of parsing / converting to raw * compare Eth2Digest using non-constant-time comparison * avoid some unnecessary validator copying This branch will in particular speed up deposit processing which has been slowing down block replay. Pre (mainnet, 1600 blocks): ``` All time are ms Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB 0.417, 0.822, 0.036, 21.098, 1400, Load block from database 16.521, 0.000, 16.521, 16.521, 1, Load state from database 27.906, 50.846, 8.104, 1507.633, 1350, Apply block 52.617, 37.029, 20.640, 135.938, 50, Apply epoch block ``` Post: ``` 3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB 0.080, 0.560, 0.035, 21.015, 1400, Load block from database 17.595, 0.000, 17.595, 17.595, 1, Load state from database 15.706, 11.028, 8.300, 107.537, 1350, Apply block 33.217, 12.622, 17.331, 60.580, 50, Apply epoch block ``` * more perf fixes * load EpochRef cache into StateCache more aggressively * point out security concern with public key cache * reuse proposer index from state when processing block * avoid genericAssign in a few more places * don't parse key when signature is unparseable * fix `==` overload for Eth2Digest * preallocate validator list when getting active validators * speed up proposer index calculation a little bit * reuse cache when replaying blocks in ncli_db * avoid a few more copying loops ``` Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB 0.072, 0.357, 0.035, 13.400, 1400, Load block from database 17.295, 0.000, 17.295, 17.295, 1, Load state from database 5.918, 9.896, 0.198, 98.028, 1350, Apply block 15.888, 10.951, 7.902, 39.535, 50, Apply epoch block 0.000, 0.000, 0.000, 0.000, 0, Database block store ``` * clear full balance cache before processing rewards and penalties ``` All time are ms Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB 0.124, 0.506, 0.026, 202.370, 363345, Load block from database 97.614, 0.000, 97.614, 97.614, 1, Load state from database 0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch 14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch 1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing 0.000, 0.000, 0.000, 0.000, 0, Database block store ```
2021-01-25 12:04:18 +00:00
# digests are already good hashes
cast[ptr Hash](unsafeAddr x.data[0])[]
2019-01-08 17:28:21 +00:00
performance fixes (#2259) * performance fixes * don't mark tree cache as dirty on read-only List accesses * store only blob in memory for keys and signatures, parse blob lazily * compare public keys by blob instead of parsing / converting to raw * compare Eth2Digest using non-constant-time comparison * avoid some unnecessary validator copying This branch will in particular speed up deposit processing which has been slowing down block replay. Pre (mainnet, 1600 blocks): ``` All time are ms Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB 0.417, 0.822, 0.036, 21.098, 1400, Load block from database 16.521, 0.000, 16.521, 16.521, 1, Load state from database 27.906, 50.846, 8.104, 1507.633, 1350, Apply block 52.617, 37.029, 20.640, 135.938, 50, Apply epoch block ``` Post: ``` 3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB 0.080, 0.560, 0.035, 21.015, 1400, Load block from database 17.595, 0.000, 17.595, 17.595, 1, Load state from database 15.706, 11.028, 8.300, 107.537, 1350, Apply block 33.217, 12.622, 17.331, 60.580, 50, Apply epoch block ``` * more perf fixes * load EpochRef cache into StateCache more aggressively * point out security concern with public key cache * reuse proposer index from state when processing block * avoid genericAssign in a few more places * don't parse key when signature is unparseable * fix `==` overload for Eth2Digest * preallocate validator list when getting active validators * speed up proposer index calculation a little bit * reuse cache when replaying blocks in ncli_db * avoid a few more copying loops ``` Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB 0.072, 0.357, 0.035, 13.400, 1400, Load block from database 17.295, 0.000, 17.295, 17.295, 1, Load state from database 5.918, 9.896, 0.198, 98.028, 1350, Apply block 15.888, 10.951, 7.902, 39.535, 50, Apply epoch block 0.000, 0.000, 0.000, 0.000, 0, Database block store ``` * clear full balance cache before processing rewards and penalties ``` All time are ms Average, StdDev, Min, Max, Samples, Test Validation is turned off meaning that no BLS operations are performed 3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB 0.124, 0.506, 0.026, 202.370, 363345, Load block from database 97.614, 0.000, 97.614, 97.614, 1, Load state from database 0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch 14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch 1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing 0.000, 0.000, 0.000, 0.000, 0, Database block store ```
2021-01-25 12:04:18 +00:00
func `==`*(a, b: Eth2Digest): bool =
# nimcrypto uses a constant-time comparison for all MDigest types which for
# Eth2Digest is unnecessary - the type should never hold a secret!
equalMem(unsafeAddr a.data[0], unsafeAddr b.data[0], sizeof(a.data))