2018-11-27 23:10:09 +00:00
|
|
|
# beacon_chain
|
2022-01-29 13:53:31 +00:00
|
|
|
# Copyright (c) 2018-2022 Status Research & Development GmbH
|
2018-11-27 23:10:09 +00:00
|
|
|
# Licensed and distributed under either of
|
2019-11-25 15:30:02 +00:00
|
|
|
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
|
|
|
|
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
|
2018-11-27 23:10:09 +00:00
|
|
|
# at your option. This file may not be copied, modified, or distributed except according to those terms.
|
|
|
|
|
|
|
|
# Serenity hash function / digest
|
|
|
|
#
|
2022-11-24 19:07:02 +00:00
|
|
|
# https://github.com/ethereum/consensus-specs/blob/v1.3.0-alpha.1/specs/phase0/beacon-chain.md#hash
|
2018-11-27 23:10:09 +00:00
|
|
|
#
|
2019-05-10 08:14:01 +00:00
|
|
|
# In Phase 0 the beacon chain is deployed with SHA256 (SHA2-256).
|
|
|
|
# Note that is is different from Keccak256 (often mistakenly called SHA3-256)
|
|
|
|
# and SHA3-256.
|
2019-03-09 20:34:08 +00:00
|
|
|
#
|
2019-05-10 08:14:01 +00:00
|
|
|
# In Eth1.0, the default hash function is Keccak256 and SHA256 is available as a precompiled contract.
|
2019-03-09 20:34:08 +00:00
|
|
|
#
|
2019-05-10 08:14:01 +00:00
|
|
|
# In our code base, to enable a smooth transition
|
|
|
|
# (already did Blake2b --> Keccak256 --> SHA2-256),
|
2020-06-16 12:16:43 +00:00
|
|
|
# we call this function `eth2digest`, and it outputs a `Eth2Digest`. Easy to sed :)
|
2018-11-27 23:10:09 +00:00
|
|
|
|
2022-07-29 10:53:42 +00:00
|
|
|
when (NimMajor, NimMinor) < (1, 4):
|
|
|
|
{.push raises: [Defect].}
|
|
|
|
else:
|
|
|
|
{.push raises: [].}
|
2020-04-11 08:51:07 +00:00
|
|
|
|
2018-12-19 12:58:53 +00:00
|
|
|
import
|
2020-09-18 14:55:55 +00:00
|
|
|
# Standard library
|
|
|
|
std/hashes,
|
2021-03-26 14:11:06 +00:00
|
|
|
# Status libraries
|
2020-04-11 08:51:07 +00:00
|
|
|
chronicles,
|
|
|
|
nimcrypto/[sha2, hash],
|
2022-11-09 22:05:50 +00:00
|
|
|
stew/[arrayops, byteutils, endians2, objects],
|
2022-06-29 16:53:59 +00:00
|
|
|
json_serialization
|
2018-11-27 23:10:09 +00:00
|
|
|
|
2022-11-09 22:05:50 +00:00
|
|
|
from nimcrypto/utils import burnMem
|
|
|
|
|
2018-12-19 12:58:53 +00:00
|
|
|
export
|
2021-03-17 10:17:15 +00:00
|
|
|
# Exports from sha2 / hash are explicit to avoid exporting upper-case `$` and
|
|
|
|
# constant-time `==`
|
2022-06-29 16:53:59 +00:00
|
|
|
hash.fromHex, json_serialization
|
2018-12-13 16:00:55 +00:00
|
|
|
|
2018-11-27 23:10:09 +00:00
|
|
|
type
|
|
|
|
Eth2Digest* = MDigest[32 * 8] ## `hash32` from spec
|
2020-09-18 14:55:55 +00:00
|
|
|
|
2022-06-29 16:53:59 +00:00
|
|
|
const PREFER_BLST_SHA256* {.booldefine.} = true
|
|
|
|
|
|
|
|
when PREFER_BLST_SHA256:
|
|
|
|
import blscurve
|
|
|
|
when BLS_BACKEND == BLST:
|
|
|
|
const USE_BLST_SHA256 = true
|
|
|
|
else:
|
|
|
|
const USE_BLST_SHA256 = false
|
|
|
|
else:
|
2022-11-09 22:05:50 +00:00
|
|
|
import nimcrypto/sha2
|
2022-06-29 16:53:59 +00:00
|
|
|
const USE_BLST_SHA256 = false
|
|
|
|
|
|
|
|
when USE_BLST_SHA256:
|
2022-11-09 22:05:50 +00:00
|
|
|
export blscurve.update, blscurve.finish
|
|
|
|
|
2020-09-18 14:55:55 +00:00
|
|
|
type Eth2DigestCtx* = BLST_SHA256_CTX
|
2022-11-09 22:05:50 +00:00
|
|
|
|
|
|
|
# HMAC support
|
|
|
|
template hmacSizeBlock*(_: type BLST_SHA256_CTX): untyped = 64
|
|
|
|
template sizeDigest*(_: BLST_SHA256_CTX): untyped = 32
|
|
|
|
|
|
|
|
proc finish*(ctx: var BLST_SHA256_CTX,
|
|
|
|
data: var openArray[byte]): uint =
|
|
|
|
var tmp {.noinit.}: array[32, byte]
|
|
|
|
finalize(tmp, ctx)
|
|
|
|
data.copyFrom(tmp).uint * 8
|
|
|
|
proc clear*(ctx: var BLST_SHA256_CTX) =
|
|
|
|
burnMem(ctx)
|
|
|
|
|
2020-09-18 14:55:55 +00:00
|
|
|
else:
|
2022-11-09 22:05:50 +00:00
|
|
|
export sha2.update, sha2.finish
|
2020-09-18 14:55:55 +00:00
|
|
|
type Eth2DigestCtx* = sha2.sha256
|
2018-11-27 23:10:09 +00:00
|
|
|
|
2021-03-17 10:17:15 +00:00
|
|
|
func `$`*(x: Eth2Digest): string =
|
|
|
|
x.data.toHex()
|
|
|
|
|
2019-02-28 21:21:29 +00:00
|
|
|
func shortLog*(x: Eth2Digest): string =
|
2022-12-12 10:12:39 +00:00
|
|
|
x.data.toHex()
|
2019-02-28 21:21:29 +00:00
|
|
|
|
2020-04-29 20:12:07 +00:00
|
|
|
chronicles.formatIt Eth2Digest:
|
|
|
|
shortLog(it)
|
|
|
|
|
2019-05-10 08:14:01 +00:00
|
|
|
# TODO: expose an in-place digest function
|
|
|
|
# when hashing in loop or into a buffer
|
|
|
|
# See: https://github.com/cheatfate/nimcrypto/blob/b90ba3abd/nimcrypto/sha2.nim#L570
|
2022-02-16 22:24:44 +00:00
|
|
|
func eth2digest*(v: openArray[byte]): Eth2Digest {.noinit.} =
|
2020-09-18 14:55:55 +00:00
|
|
|
## Apply the Eth2 Hash function
|
|
|
|
## Do NOT use for secret data.
|
2022-06-29 16:53:59 +00:00
|
|
|
when USE_BLST_SHA256:
|
2020-09-18 14:55:55 +00:00
|
|
|
# BLST has a fast assembly optimized SHA256
|
|
|
|
result.data.bls_sha256_digest(v)
|
|
|
|
else:
|
|
|
|
# We use the init-update-finish interface to avoid
|
|
|
|
# the expensive burning/clearing memory (20~30% perf)
|
2022-06-29 16:53:59 +00:00
|
|
|
var ctx {.noinit.}: Eth2DigestCtx
|
2020-09-18 14:55:55 +00:00
|
|
|
ctx.init()
|
|
|
|
ctx.update(v)
|
|
|
|
ctx.finish()
|
2018-11-27 23:10:09 +00:00
|
|
|
|
|
|
|
template withEth2Hash*(body: untyped): Eth2Digest =
|
|
|
|
## This little helper will init the hash function and return the sliced
|
|
|
|
## hash:
|
|
|
|
## let hashOfData = withHash: h.update(data)
|
2020-09-18 14:55:55 +00:00
|
|
|
when nimvm:
|
|
|
|
# In SSZ, computeZeroHashes require compile-time SHA256
|
|
|
|
block:
|
|
|
|
var h {.inject.}: sha256
|
|
|
|
init(h)
|
|
|
|
body
|
|
|
|
finish(h)
|
|
|
|
else:
|
2022-06-29 16:53:59 +00:00
|
|
|
when USE_BLST_SHA256:
|
2020-09-18 14:55:55 +00:00
|
|
|
block:
|
2022-06-29 16:53:59 +00:00
|
|
|
var h {.inject, noinit.}: Eth2DigestCtx
|
2020-09-18 14:55:55 +00:00
|
|
|
init(h)
|
|
|
|
body
|
2022-02-16 22:24:44 +00:00
|
|
|
var res {.noinit.}: Eth2Digest
|
2020-09-18 14:55:55 +00:00
|
|
|
finalize(res.data, h)
|
|
|
|
res
|
|
|
|
else:
|
|
|
|
block:
|
2022-06-29 16:53:59 +00:00
|
|
|
var h {.inject, noinit.}: Eth2DigestCtx
|
2020-09-18 14:55:55 +00:00
|
|
|
init(h)
|
|
|
|
body
|
|
|
|
finish(h)
|
2019-01-08 17:28:21 +00:00
|
|
|
|
performance fixes (#2259)
* performance fixes
* don't mark tree cache as dirty on read-only List accesses
* store only blob in memory for keys and signatures, parse blob lazily
* compare public keys by blob instead of parsing / converting to raw
* compare Eth2Digest using non-constant-time comparison
* avoid some unnecessary validator copying
This branch will in particular speed up deposit processing which has
been slowing down block replay.
Pre (mainnet, 1600 blocks):
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB
0.417, 0.822, 0.036, 21.098, 1400, Load block from database
16.521, 0.000, 16.521, 16.521, 1, Load state from database
27.906, 50.846, 8.104, 1507.633, 1350, Apply block
52.617, 37.029, 20.640, 135.938, 50, Apply epoch block
```
Post:
```
3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB
0.080, 0.560, 0.035, 21.015, 1400, Load block from database
17.595, 0.000, 17.595, 17.595, 1, Load state from database
15.706, 11.028, 8.300, 107.537, 1350, Apply block
33.217, 12.622, 17.331, 60.580, 50, Apply epoch block
```
* more perf fixes
* load EpochRef cache into StateCache more aggressively
* point out security concern with public key cache
* reuse proposer index from state when processing block
* avoid genericAssign in a few more places
* don't parse key when signature is unparseable
* fix `==` overload for Eth2Digest
* preallocate validator list when getting active validators
* speed up proposer index calculation a little bit
* reuse cache when replaying blocks in ncli_db
* avoid a few more copying loops
```
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB
0.072, 0.357, 0.035, 13.400, 1400, Load block from database
17.295, 0.000, 17.295, 17.295, 1, Load state from database
5.918, 9.896, 0.198, 98.028, 1350, Apply block
15.888, 10.951, 7.902, 39.535, 50, Apply epoch block
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
* clear full balance cache before processing rewards and penalties
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB
0.124, 0.506, 0.026, 202.370, 363345, Load block from database
97.614, 0.000, 97.614, 97.614, 1, Load state from database
0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch
14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch
1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
2021-01-25 12:04:18 +00:00
|
|
|
template hash*(x: Eth2Digest): Hash =
|
2019-05-10 08:14:01 +00:00
|
|
|
## Hash for digests for Nim hash tables
|
performance fixes (#2259)
* performance fixes
* don't mark tree cache as dirty on read-only List accesses
* store only blob in memory for keys and signatures, parse blob lazily
* compare public keys by blob instead of parsing / converting to raw
* compare Eth2Digest using non-constant-time comparison
* avoid some unnecessary validator copying
This branch will in particular speed up deposit processing which has
been slowing down block replay.
Pre (mainnet, 1600 blocks):
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB
0.417, 0.822, 0.036, 21.098, 1400, Load block from database
16.521, 0.000, 16.521, 16.521, 1, Load state from database
27.906, 50.846, 8.104, 1507.633, 1350, Apply block
52.617, 37.029, 20.640, 135.938, 50, Apply epoch block
```
Post:
```
3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB
0.080, 0.560, 0.035, 21.015, 1400, Load block from database
17.595, 0.000, 17.595, 17.595, 1, Load state from database
15.706, 11.028, 8.300, 107.537, 1350, Apply block
33.217, 12.622, 17.331, 60.580, 50, Apply epoch block
```
* more perf fixes
* load EpochRef cache into StateCache more aggressively
* point out security concern with public key cache
* reuse proposer index from state when processing block
* avoid genericAssign in a few more places
* don't parse key when signature is unparseable
* fix `==` overload for Eth2Digest
* preallocate validator list when getting active validators
* speed up proposer index calculation a little bit
* reuse cache when replaying blocks in ncli_db
* avoid a few more copying loops
```
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB
0.072, 0.357, 0.035, 13.400, 1400, Load block from database
17.295, 0.000, 17.295, 17.295, 1, Load state from database
5.918, 9.896, 0.198, 98.028, 1350, Apply block
15.888, 10.951, 7.902, 39.535, 50, Apply epoch block
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
* clear full balance cache before processing rewards and penalties
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB
0.124, 0.506, 0.026, 202.370, 363345, Load block from database
97.614, 0.000, 97.614, 97.614, 1, Load state from database
0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch
14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch
1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
2021-01-25 12:04:18 +00:00
|
|
|
# digests are already good hashes
|
|
|
|
cast[ptr Hash](unsafeAddr x.data[0])[]
|
2019-01-08 17:28:21 +00:00
|
|
|
|
performance fixes (#2259)
* performance fixes
* don't mark tree cache as dirty on read-only List accesses
* store only blob in memory for keys and signatures, parse blob lazily
* compare public keys by blob instead of parsing / converting to raw
* compare Eth2Digest using non-constant-time comparison
* avoid some unnecessary validator copying
This branch will in particular speed up deposit processing which has
been slowing down block replay.
Pre (mainnet, 1600 blocks):
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3450.269, 0.000, 3450.269, 3450.269, 1, Initialize DB
0.417, 0.822, 0.036, 21.098, 1400, Load block from database
16.521, 0.000, 16.521, 16.521, 1, Load state from database
27.906, 50.846, 8.104, 1507.633, 1350, Apply block
52.617, 37.029, 20.640, 135.938, 50, Apply epoch block
```
Post:
```
3502.715, 0.000, 3502.715, 3502.715, 1, Initialize DB
0.080, 0.560, 0.035, 21.015, 1400, Load block from database
17.595, 0.000, 17.595, 17.595, 1, Load state from database
15.706, 11.028, 8.300, 107.537, 1350, Apply block
33.217, 12.622, 17.331, 60.580, 50, Apply epoch block
```
* more perf fixes
* load EpochRef cache into StateCache more aggressively
* point out security concern with public key cache
* reuse proposer index from state when processing block
* avoid genericAssign in a few more places
* don't parse key when signature is unparseable
* fix `==` overload for Eth2Digest
* preallocate validator list when getting active validators
* speed up proposer index calculation a little bit
* reuse cache when replaying blocks in ncli_db
* avoid a few more copying loops
```
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3279.158, 0.000, 3279.158, 3279.158, 1, Initialize DB
0.072, 0.357, 0.035, 13.400, 1400, Load block from database
17.295, 0.000, 17.295, 17.295, 1, Load state from database
5.918, 9.896, 0.198, 98.028, 1350, Apply block
15.888, 10.951, 7.902, 39.535, 50, Apply epoch block
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
* clear full balance cache before processing rewards and penalties
```
All time are ms
Average, StdDev, Min, Max, Samples, Test
Validation is turned off meaning that no BLS operations are performed
3947.901, 0.000, 3947.901, 3947.901, 1, Initialize DB
0.124, 0.506, 0.026, 202.370, 363345, Load block from database
97.614, 0.000, 97.614, 97.614, 1, Load state from database
0.186, 0.188, 0.012, 99.561, 357262, Advance slot, non-epoch
14.161, 5.966, 1.099, 395.511, 11524, Advance slot, epoch
1.372, 4.170, 0.017, 276.401, 363345, Apply block, no slot processing
0.000, 0.000, 0.000, 0.000, 0, Database block store
```
2021-01-25 12:04:18 +00:00
|
|
|
func `==`*(a, b: Eth2Digest): bool =
|
2021-02-10 04:02:04 +00:00
|
|
|
when nimvm:
|
2021-02-10 13:22:08 +00:00
|
|
|
a.data == b.data
|
2021-02-10 04:02:04 +00:00
|
|
|
else:
|
|
|
|
# nimcrypto uses a constant-time comparison for all MDigest types which for
|
|
|
|
# Eth2Digest is unnecessary - the type should never hold a secret!
|
|
|
|
equalMem(unsafeAddr a.data[0], unsafeAddr b.data[0], sizeof(a.data))
|
2021-03-26 14:11:06 +00:00
|
|
|
|
2022-02-14 05:26:19 +00:00
|
|
|
func isZero*(x: Eth2Digest): bool =
|
|
|
|
x.isZeroMemory
|
|
|
|
|
2021-03-26 14:11:06 +00:00
|
|
|
proc writeValue*(w: var JsonWriter, a: Eth2Digest) {.raises: [Defect, IOError, SerializationError].} =
|
|
|
|
w.writeValue $a
|
|
|
|
|
|
|
|
proc readValue*(r: var JsonReader, a: var Eth2Digest) {.raises: [Defect, IOError, SerializationError].} =
|
|
|
|
try:
|
|
|
|
a = fromHex(type(a), r.readValue(string))
|
|
|
|
except ValueError:
|
|
|
|
raiseUnexpectedValue(r, "Hex string expected")
|
2021-08-12 13:08:20 +00:00
|
|
|
|
2022-02-17 11:53:55 +00:00
|
|
|
func toGaugeValue*(hash: Eth2Digest): int64 =
|
2021-08-12 13:08:20 +00:00
|
|
|
# Only the last 8 bytes are taken into consideration in accordance
|
|
|
|
# to the ETH2 metrics spec:
|
2022-06-09 14:30:13 +00:00
|
|
|
# https://github.com/ethereum/beacon-metrics/blob/6a79914cb31f7d54858c7dd57eee75b6162ec737/metrics.md#interop-metrics
|
2021-08-12 13:08:20 +00:00
|
|
|
cast[int64](uint64.fromBytesLE(hash.data.toOpenArray(24, 31)))
|