nimbus-eth1/nimbus/utils/utils.nim

143 lines
4.5 KiB
Nim
Raw Normal View History

2023-11-01 03:32:09 +00:00
# Nimbus
# Copyright (c) 2018-2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
Aristo db update for short nodes key edge cases (#1887) * Aristo: Provide key-value list signature calculator detail: Simple wrappers around `Aristo` core functionality * Update new API for `CoreDb` details: + Renamed new API functions `contains()` => `hasKey()` or `hasPath()` which disables the `in` operator on non-boolean `contains()` functions + The functions `get()` and `fetch()` always return a not-found error if there is no item, available. The new functions `getOrEmpty()` and `mergeOrEmpty()` return an an empty `Blob` if there is no such key found. * Rewrite `core_apps.nim` using new API from `CoreDb` * Use `Aristo` functionality for calculating Merkle signatures details: For debugging, the `VerifyAristoForMerkleRootCalc` can be set so that `Aristo` results will be verified against the legacy versions. * Provide general interface for Merkle signing key-value tables details: Export `Aristo` wrappers * Activate `CoreDb` tests why: Now, API seems to be stable enough for general tests. * Update `toHex()` usage why: Byteutils' `toHex()` is superior to `toSeq.mapIt(it.toHex(2)).join` * Split `aristo_transcode` => `aristo_serialise` + `aristo_blobify` why: + Different modules for different purposes + `aristo_serialise`: RLP encoding/decoding + `aristo_blobify`: Aristo database encoding/decoding * Compacted representation of small nodes' links instead of Keccak hashes why: Ethereum MPTs use Keccak hashes as node links if the size of an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded node value is used as a pseudo node link (rather than a hash.) Such a node is nor stored on key-value database. Rather the RLP encoded node value is stored instead of a lode link in a parent node instead. Only for the root hash, the top level node is always referred to by the hash. This feature needed an abstraction of the `HashKey` object which is now either a hash or a blob of length at most 31 bytes. This leaves two ways of representing an empty/void `HashKey` type, either as an empty blob of zero length, or the hash of an empty blob. * Update `CoreDb` interface (mainly reducing logger noise) * Fix copyright years (to make `Lint` happy)
2023-11-08 12:18:32 +00:00
{.push raises: [].}
import
std/[math, times, strutils],
Unified database frontend integration (#1670) * Nimbus folder environment update details: * Integrated `CoreDbRef` for the sources in the `nimbus` sub-folder. * The `nimbus` program does not compile yet as it needs the updates in the parallel `stateless` sub-folder. * Stateless environment update details: * Integrated `CoreDbRef` for the sources in the `stateless` sub-folder. * The `nimbus` program compiles now. * Premix environment update details: * Integrated `CoreDbRef` for the sources in the `premix` sub-folder. * Fluffy environment update details: * Integrated `CoreDbRef` for the sources in the `fluffy` sub-folder. * Tools environment update details: * Integrated `CoreDbRef` for the sources in the `tools` sub-folder. * Nodocker environment update details: * Integrated `CoreDbRef` for the sources in the `hive_integration/nodocker` sub-folder. * Tests environment update details: * Integrated `CoreDbRef` for the sources in the `tests` sub-folder. * The unit tests compile and run cleanly now. * Generalise `CoreDbRef` to any `select_backend` supported database why: Generalisation was just missed due to overcoming some compiler oddity which was tied to rocksdb for testing. * Suppress compiler warning for `newChainDB()` why: Warning was added to this function which must be wrapped so that any `CatchableError` is re-raised as `Defect`. * Split off persistent `CoreDbRef` constructor into separate file why: This allows to compile a memory only database version without linking the backend library. * Use memory `CoreDbRef` database by default detail: Persistent DB constructor needs to import `db/core_db/persistent why: Most tests use memory DB anyway. This avoids linking `-lrocksdb` or any other backend by default. * fix `toLegacyBackend()` availability check why: got garbled after memory/persistent split. * Clarify raw access to MPT for snap sync handler why: Logically, `kvt` is not the raw access for the hexary trie (although this holds for the legacy database)
2023-08-04 11:10:09 +00:00
eth/[rlp, common/eth_types_rlp],
stew/byteutils,
nimcrypto,
Aristo db update for short nodes key edge cases (#1887) * Aristo: Provide key-value list signature calculator detail: Simple wrappers around `Aristo` core functionality * Update new API for `CoreDb` details: + Renamed new API functions `contains()` => `hasKey()` or `hasPath()` which disables the `in` operator on non-boolean `contains()` functions + The functions `get()` and `fetch()` always return a not-found error if there is no item, available. The new functions `getOrEmpty()` and `mergeOrEmpty()` return an an empty `Blob` if there is no such key found. * Rewrite `core_apps.nim` using new API from `CoreDb` * Use `Aristo` functionality for calculating Merkle signatures details: For debugging, the `VerifyAristoForMerkleRootCalc` can be set so that `Aristo` results will be verified against the legacy versions. * Provide general interface for Merkle signing key-value tables details: Export `Aristo` wrappers * Activate `CoreDb` tests why: Now, API seems to be stable enough for general tests. * Update `toHex()` usage why: Byteutils' `toHex()` is superior to `toSeq.mapIt(it.toHex(2)).join` * Split `aristo_transcode` => `aristo_serialise` + `aristo_blobify` why: + Different modules for different purposes + `aristo_serialise`: RLP encoding/decoding + `aristo_blobify`: Aristo database encoding/decoding * Compacted representation of small nodes' links instead of Keccak hashes why: Ethereum MPTs use Keccak hashes as node links if the size of an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded node value is used as a pseudo node link (rather than a hash.) Such a node is nor stored on key-value database. Rather the RLP encoded node value is stored instead of a lode link in a parent node instead. Only for the root hash, the top level node is always referred to by the hash. This feature needed an abstraction of the `HashKey` object which is now either a hash or a blob of length at most 31 bytes. This leaves two ways of representing an empty/void `HashKey` type, either as an empty blob of zero length, or the hash of an empty blob. * Update `CoreDb` interface (mainly reducing logger noise) * Fix copyright years (to make `Lint` happy)
2023-11-08 12:18:32 +00:00
results,
../db/core_db,
../constants
2018-12-03 10:54:19 +00:00
export eth_types_rlp
2019-02-26 11:26:38 +00:00
proc calcRootHash[T](items: openArray[T]): Hash256 {.gcsafe.} =
Aristo db update for short nodes key edge cases (#1887) * Aristo: Provide key-value list signature calculator detail: Simple wrappers around `Aristo` core functionality * Update new API for `CoreDb` details: + Renamed new API functions `contains()` => `hasKey()` or `hasPath()` which disables the `in` operator on non-boolean `contains()` functions + The functions `get()` and `fetch()` always return a not-found error if there is no item, available. The new functions `getOrEmpty()` and `mergeOrEmpty()` return an an empty `Blob` if there is no such key found. * Rewrite `core_apps.nim` using new API from `CoreDb` * Use `Aristo` functionality for calculating Merkle signatures details: For debugging, the `VerifyAristoForMerkleRootCalc` can be set so that `Aristo` results will be verified against the legacy versions. * Provide general interface for Merkle signing key-value tables details: Export `Aristo` wrappers * Activate `CoreDb` tests why: Now, API seems to be stable enough for general tests. * Update `toHex()` usage why: Byteutils' `toHex()` is superior to `toSeq.mapIt(it.toHex(2)).join` * Split `aristo_transcode` => `aristo_serialise` + `aristo_blobify` why: + Different modules for different purposes + `aristo_serialise`: RLP encoding/decoding + `aristo_blobify`: Aristo database encoding/decoding * Compacted representation of small nodes' links instead of Keccak hashes why: Ethereum MPTs use Keccak hashes as node links if the size of an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded node value is used as a pseudo node link (rather than a hash.) Such a node is nor stored on key-value database. Rather the RLP encoded node value is stored instead of a lode link in a parent node instead. Only for the root hash, the top level node is always referred to by the hash. This feature needed an abstraction of the `HashKey` object which is now either a hash or a blob of length at most 31 bytes. This leaves two ways of representing an empty/void `HashKey` type, either as an empty blob of zero length, or the hash of an empty blob. * Update `CoreDb` interface (mainly reducing logger noise) * Fix copyright years (to make `Lint` happy)
2023-11-08 12:18:32 +00:00
let sig = merkleSignBegin()
2018-12-03 10:54:19 +00:00
for i, t in items:
Aristo db update for short nodes key edge cases (#1887) * Aristo: Provide key-value list signature calculator detail: Simple wrappers around `Aristo` core functionality * Update new API for `CoreDb` details: + Renamed new API functions `contains()` => `hasKey()` or `hasPath()` which disables the `in` operator on non-boolean `contains()` functions + The functions `get()` and `fetch()` always return a not-found error if there is no item, available. The new functions `getOrEmpty()` and `mergeOrEmpty()` return an an empty `Blob` if there is no such key found. * Rewrite `core_apps.nim` using new API from `CoreDb` * Use `Aristo` functionality for calculating Merkle signatures details: For debugging, the `VerifyAristoForMerkleRootCalc` can be set so that `Aristo` results will be verified against the legacy versions. * Provide general interface for Merkle signing key-value tables details: Export `Aristo` wrappers * Activate `CoreDb` tests why: Now, API seems to be stable enough for general tests. * Update `toHex()` usage why: Byteutils' `toHex()` is superior to `toSeq.mapIt(it.toHex(2)).join` * Split `aristo_transcode` => `aristo_serialise` + `aristo_blobify` why: + Different modules for different purposes + `aristo_serialise`: RLP encoding/decoding + `aristo_blobify`: Aristo database encoding/decoding * Compacted representation of small nodes' links instead of Keccak hashes why: Ethereum MPTs use Keccak hashes as node links if the size of an RLP encoded node is at least 32 bytes. Otherwise, the RLP encoded node value is used as a pseudo node link (rather than a hash.) Such a node is nor stored on key-value database. Rather the RLP encoded node value is stored instead of a lode link in a parent node instead. Only for the root hash, the top level node is always referred to by the hash. This feature needed an abstraction of the `HashKey` object which is now either a hash or a blob of length at most 31 bytes. This leaves two ways of representing an empty/void `HashKey` type, either as an empty blob of zero length, or the hash of an empty blob. * Update `CoreDb` interface (mainly reducing logger noise) * Fix copyright years (to make `Lint` happy)
2023-11-08 12:18:32 +00:00
sig.merkleSignAdd(rlp.encode(i), rlp.encode(t))
sig.merkleSignCommit.value.to(Hash256)
2018-12-03 10:54:19 +00:00
template calcTxRoot*(transactions: openArray[Transaction]): Hash256 =
calcRootHash(transactions)
template calcWithdrawalsRoot*(withdrawals: openArray[Withdrawal]): Hash256 =
calcRootHash(withdrawals)
2018-12-03 10:54:19 +00:00
template calcReceiptRoot*(receipts: openArray[Receipt]): Hash256 =
calcRootHash(receipts)
2019-02-26 07:04:12 +00:00
func sumHash*(hashes: varargs[Hash256]): Hash256 =
var ctx: sha256
ctx.init()
for hash in hashes:
ctx.update hash.data
ctx.finish result.data
ctx.clear()
proc sumHash*(body: BlockBody): Hash256 {.gcsafe, raises: []} =
let txRoot = calcTxRoot(body.transactions)
let ommersHash = keccakHash(rlp.encode(body.uncles))
let wdRoot = if body.withdrawals.isSome:
calcWithdrawalsRoot(body.withdrawals.get)
else: EMPTY_ROOT_HASH
sumHash(txRoot, ommersHash, wdRoot)
proc sumHash*(header: BlockHeader): Hash256 =
let wdRoot = if header.withdrawalsRoot.isSome:
header.withdrawalsRoot.get
else: EMPTY_ROOT_HASH
sumHash(header.txRoot, header.ommersHash, wdRoot)
func hasBody*(h: BlockHeader): bool =
h.txRoot != EMPTY_ROOT_HASH or
h.ommersHash != EMPTY_UNCLE_HASH or
h.withdrawalsRoot.get(EMPTY_ROOT_HASH) != EMPTY_ROOT_HASH
2019-02-26 07:04:12 +00:00
func generateAddress*(address: EthAddress, nonce: AccountNonce): EthAddress =
2019-03-07 15:53:09 +00:00
result[0..19] = keccakHash(rlp.encodeList(address, nonce)).data.toOpenArray(12, 31)
2019-02-26 07:04:12 +00:00
Transaction: EVMC fix, `CREATE2` salt is a 256-bit blob not a number This changes fixes a bug in `CREATE2` ops when used with EVMC. Because it changes the salt type, it affects non-EVMC code as well. The salt was passed through EVMC with the wrong byte order, although this went unnoticed as the Nimbus host flipped the byte order before using it. This was found when running Nimbus with third-party EVM, ["evmone"](https://github.com/ethereum/evmone). There are different ways to remedy this. If treated as a number, Nimbus EVM would byte-flip the value when calling EVMC, then Nimbus host would flip the received value. Finally, it would be flipped a third time when generating the address in `generateSafeAddress`. The first two flips can be eliminated by negotiation (like other numbers), but there would always be one flip. As a bit pattern, Nimbus EVM would flip the same way it does when dealing with hashes on the stack (e.g. with `getBlockHash`). Nimbus host wouldn't flip at all - and when using third-party EVMs there would be no flips in Nimbus. Because this value is not for arithmetic, any bit pattern is valid, and there shouldn't be any flips when using a third-party EVM, the bit-pattern interpretation is favoured. The only flip is done in Nimbus EVM (and might be eliminated in an optimised version). As suggested, we'll define a new "opaque 256 bits" type to hold this value. (Similar to `Hash256`, but the salt isn't necessarily a hash.) Signed-off-by: Jamie Lokier <jamie@shareable.org>
2021-06-09 22:32:42 +00:00
type ContractSalt* = object
bytes*: array[32, byte]
const ZERO_CONTRACTSALT* = default(ContractSalt)
func generateSafeAddress*(address: EthAddress, salt: ContractSalt,
data: openArray[byte]): EthAddress =
2019-05-07 03:23:27 +00:00
const prefix = [0xff.byte]
let
dataHash = keccakHash(data)
hashResult = withKeccakHash:
h.update(prefix)
h.update(address)
h.update(salt.bytes)
h.update(dataHash.data)
2019-05-07 03:23:27 +00:00
result[0..19] = hashResult.data.toOpenArray(12, 31)
2019-02-26 07:04:12 +00:00
func hash*(b: BlockHeader): Hash256 {.inline.} =
rlpHash(b)
proc crc32*(crc: uint32, buf: openArray[byte]): uint32 =
const kcrc32 = [ 0'u32, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190,
0x6b6b51f4, 0x4db26158, 0x5005713c, 0xedb88320'u32, 0xf00f9344'u32, 0xd6d6a3e8'u32,
0xcb61b38c'u32, 0x9b64c2b0'u32, 0x86d3d2d4'u32, 0xa00ae278'u32, 0xbdbdf21c'u32]
var crcu32 = not crc
for b in buf:
crcu32 = (crcu32 shr 4) xor kcrc32[int((crcu32 and 0xF) xor (uint32(b) and 0xF'u32))]
crcu32 = (crcu32 shr 4) xor kcrc32[int((crcu32 and 0xF) xor (uint32(b) shr 4'u32))]
result = not crcu32
proc short*(h: Hash256): string =
var bytes: array[6, byte]
bytes[0..2] = h.data[0..2]
bytes[^3..^1] = h.data[^3..^1]
bytes.toHex
2023-05-30 08:58:41 +00:00
func short*(x: Duration): string =
let parts = x.toParts
if parts[Hours] > 0:
result.add $parts[Hours]
result.add ':'
result.add intToStr(parts[Minutes].int, 2)
result.add ':'
result.add intToStr(parts[Seconds].int, 2)
proc decompose*(rlp: var Rlp,
header: var BlockHeader,
2023-05-30 08:58:41 +00:00
body: var BlockBody) {.gcsafe, raises: [RlpError].} =
var blk = rlp.read(EthBlock)
header = system.move(blk.header)
body.transactions = system.move(blk.txs)
body.uncles = system.move(blk.uncles)
body.withdrawals = system.move(blk.withdrawals)
proc decompose*(rlpBytes: openArray[byte],
header: var BlockHeader,
2023-05-30 08:58:41 +00:00
body: var BlockBody) {.gcsafe, raises: [RlpError].} =
var rlp = rlpFromBytes(rlpBytes)
rlp.decompose(header, body)
func gwei*(n: uint64): GasInt =
GasInt(n * (10'u64 ^ 9'u64))
# Helper types to convert gwei into wei more easily
func weiAmount*(w: Withdrawal): UInt256 =
w.amount.u256 * (10'u64 ^ 9'u64).u256
func isGenesis*(header: BlockHeader): bool =
header.blockNumber == 0.u256 and
header.parentHash == GENESIS_PARENT_HASH