From c4c37302b121b81cc5952fa921206e66dbf693b9 Mon Sep 17 00:00:00 2001 From: Etan Kissling Date: Wed, 15 May 2024 06:07:59 +0300 Subject: [PATCH] Introduce wrapper type for EIP-4844 transactions (#2177) * Introduce wrapper type for EIP-4844 transactions EIP-4844 blob sidecars are a concept that only exists in the mempool. After inclusion of a transaction into an execution block, only the versioned hash within the transaction remains. To improve type safety, replace the `Transaction.networkPayload` member with a wrapper type `PooledTransaction` that is used in contexts where blob sidecars exist. * Bump nimbus-eth2 to 87605d08a7f9cfc3b223bd32143e93a6cdf351ac * IPv6 'listen-address' in `nimbus_verified_proxy` * Bump nim-libp2p to 21cbe3a91a70811522554e89e6a791172cebfef2 * Fix beacon_lc_bridge payload conversion and conf.listenAddress type * Change nimbus_verified_proxy.asExecutionData param to SomeExecutionPayload * Rerun nph to fix asExecutionData style format * nimbus_verified_proxy listenAddress * Use PooledTransaction in nimbus-eth1 tests --------- Co-authored-by: jangko --- .../beacon_lc_bridge/beacon_lc_bridge.nim | 2 +- .../beacon_lc_bridge_conf.nim | 5 +- .../nodocker/engine/cancun/customizer.nim | 2 +- .../nodocker/engine/cancun/helpers.nim | 12 +- .../engine/cancun/step_devp2p_pooledtx.nim | 4 +- .../engine/cancun/step_sendblobtx.nim | 2 +- hive_integration/nodocker/engine/clmock.nim | 2 +- .../engine/engine/invalid_ancestor.nim | 2 +- .../engine/engine/invalid_payload.nim | 6 +- .../nodocker/engine/engine/prev_randao.nim | 4 +- .../nodocker/engine/engine/reorg.nim | 10 +- .../nodocker/engine/engine_client.nim | 8 +- hive_integration/nodocker/engine/test_env.nim | 35 +++-- .../nodocker/engine/tx_sender.nim | 127 +++++++++++------- .../withdrawals/wd_max_init_code_spec.nim | 2 +- hive_integration/nodocker/rpc/client.nim | 3 +- hive_integration/nodocker/rpc/rpc_tests.nim | 8 +- hive_integration/nodocker/rpc/vault.nim | 11 +- nimbus/beacon/api_handler/api_forkchoice.nim | 8 +- nimbus/beacon/api_handler/api_getpayload.nim | 56 +++----- nimbus/beacon/api_handler/api_newpayload.nim | 12 +- nimbus/beacon/beacon_engine.nim | 53 ++++++-- nimbus/beacon/payload_conv.nim | 22 ++- nimbus/beacon/payload_queue.nim | 76 +++++++---- nimbus/beacon/web3_eth_conv.nim | 10 +- nimbus/core/eip4844.nim | 15 ++- nimbus/core/sealer.nim | 5 +- nimbus/core/tx_pool.nim | 48 +++++-- nimbus/core/tx_pool/tx_item.nim | 18 ++- nimbus/core/tx_pool/tx_tabs.nim | 8 +- nimbus/core/tx_pool/tx_tasks/tx_add.nim | 6 +- nimbus/core/tx_pool/tx_tasks/tx_classify.nim | 5 +- nimbus/core/tx_pool/tx_tasks/tx_head.nim | 12 +- nimbus/core/tx_pool/tx_tasks/tx_packer.nim | 2 +- nimbus/core/tx_pool/tx_tasks/tx_recover.nim | 4 +- nimbus/core/validate.nim | 3 - nimbus/db/core_db/core_apps_newapi.nim | 4 +- nimbus/graphql/ethapi.nim | 4 +- nimbus/rpc/p2p.nim | 29 +++- nimbus/sync/handlers/eth.nim | 12 +- nimbus/sync/protocol/eth/eth_types.nim | 2 +- nimbus/sync/protocol/eth66.nim | 3 +- nimbus/sync/protocol/eth67.nim | 3 +- nimbus/sync/protocol/eth68.nim | 3 +- nimbus/transaction.nim | 6 + nimbus/utils/debug.nim | 9 +- .../libverifproxy/verifproxy.nim | 3 +- .../nimbus_verified_proxy_conf.nim | 4 +- nimbus_verified_proxy/rpc/rpc_utils.nim | 4 +- tests/replay/pp.nim | 1 - tests/test_eip4844.nim | 11 +- tests/test_txpool.nim | 8 +- tests/test_txpool/setup.nim | 14 +- tests/test_txpool2.nim | 24 ++-- vendor/nim-eth | 2 +- vendor/nim-libp2p | 2 +- vendor/nimbus-eth2 | 2 +- 57 files changed, 450 insertions(+), 308 deletions(-) diff --git a/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim b/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim index cd9725027..e6cb9d876 100644 --- a/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim +++ b/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge.nim @@ -154,7 +154,7 @@ proc asPortalBlockData*( (hash, headerWithProof, body) proc asPortalBlockData*( - payload: ExecutionPayloadV2 | ExecutionPayloadV3 + payload: ExecutionPayloadV2 | ExecutionPayloadV3 | ExecutionPayloadV4 ): (common_types.BlockHash, BlockHeaderWithProof, PortalBlockBodyShanghai) = let txRoot = calculateTransactionData(payload.transactions) diff --git a/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge_conf.nim b/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge_conf.nim index 5528d9754..282e07cb4 100644 --- a/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge_conf.nim +++ b/fluffy/tools/beacon_lc_bridge/beacon_lc_bridge_conf.nim @@ -99,10 +99,9 @@ type BeaconBridgeConf* = object # Config listenAddress* {. desc: "Listening address for the Ethereum LibP2P and Discovery v5 traffic", - defaultValue: defaultListenAddress, - defaultValueDesc: $defaultListenAddressDesc, + defaultValueDesc: "*", name: "listen-address" - .}: IpAddress + .}: Option[IpAddress] tcpPort* {. desc: "Listening TCP port for Ethereum LibP2P traffic", diff --git a/hive_integration/nodocker/engine/cancun/customizer.nim b/hive_integration/nodocker/engine/cancun/customizer.nim index 037f1c2f7..4f874ba02 100644 --- a/hive_integration/nodocker/engine/cancun/customizer.nim +++ b/hive_integration/nodocker/engine/cancun/customizer.nim @@ -336,7 +336,7 @@ func getTimestamp*(cust: CustomPayloadData, basePayload: ExecutionPayload): uint # Construct a customized payload by taking an existing payload as base and mixing it CustomPayloadData # blockHash is calculated automatically. proc customizePayload*(cust: CustomPayloadData, data: ExecutableData): ExecutableData {.gcsafe.} = - var customHeader = blockHeader(data.basePayload, removeBlobs = false, beaconRoot = data.beaconRoot) + var customHeader = blockHeader(data.basePayload, beaconRoot = data.beaconRoot) if cust.transactions.isSome: customHeader.txRoot = calcTxRoot(cust.transactions.get) diff --git a/hive_integration/nodocker/engine/cancun/helpers.nim b/hive_integration/nodocker/engine/cancun/helpers.nim index 3a1f1dbfa..7ebabe312 100644 --- a/hive_integration/nodocker/engine/cancun/helpers.nim +++ b/hive_integration/nodocker/engine/cancun/helpers.nim @@ -29,7 +29,7 @@ type TestBlobTxPool* = ref object currentBlobID* : BlobID currentTxIndex*: int - transactions* : Table[common.Hash256, Transaction] + transactions* : Table[common.Hash256, PooledTransaction] hashesByIndex* : Table[int, common.Hash256] const @@ -53,7 +53,7 @@ func getMinExcessBlobGasForBlobGasPrice(data_gas_price: uint64): uint64 = func getMinExcessBlobsForBlobGasPrice*(data_gas_price: uint64): uint64 = return getMinExcessBlobGasForBlobGasPrice(data_gas_price) div GAS_PER_BLOB.uint64 -proc addBlobTransaction*(pool: TestBlobTxPool, tx: Transaction) = +proc addBlobTransaction*(pool: TestBlobTxPool, tx: PooledTransaction) = let txHash = rlpHash(tx) pool.transactions[txHash] = tx @@ -178,19 +178,19 @@ proc getBlobDataInPayload*(pool: TestBlobTxPool, payload: ExecutionPayload): Res return err("blob data is nil") let np = blobTx.networkPayload - if blobTx.versionedHashes.len != np.commitments.len or + if blobTx.tx.versionedHashes.len != np.commitments.len or np.commitments.len != np.blobs.len or np.blobs.len != np.proofs.len: return err("invalid blob wrap data") - for i in 0.. 0: # Reasoning: Most of the clients do not re-add blob transactions to the pool diff --git a/hive_integration/nodocker/engine/engine_client.nim b/hive_integration/nodocker/engine/engine_client.nim index 731c3c95b..b4b8609a7 100644 --- a/hive_integration/nodocker/engine/engine_client.nim +++ b/hive_integration/nodocker/engine/engine_client.nim @@ -512,7 +512,8 @@ proc namedHeader*(client: RpcClient, name: string): Result[common.BlockHeader, s return err("failed to get named blockHeader") return ok(res.toBlockHeader) -proc sendTransaction*(client: RpcClient, tx: common.Transaction): Result[void, string] = +proc sendTransaction*( + client: RpcClient, tx: common.PooledTransaction): Result[void, string] = wrapTry: let encodedTx = rlp.encode(tx) let res = waitFor client.eth_sendRawTransaction(encodedTx) @@ -603,7 +604,10 @@ TraceOpts.useDefaultSerializationIn JrpcConv createRpcSigsFromNim(RpcClient): proc debug_traceTransaction(hash: TxHash, opts: TraceOpts): JsonNode -proc debugPrevRandaoTransaction*(client: RpcClient, tx: Transaction, expectedPrevRandao: Hash256): Result[void, string] = +proc debugPrevRandaoTransaction*( + client: RpcClient, + tx: PooledTransaction, + expectedPrevRandao: Hash256): Result[void, string] = wrapTry: let hash = w3Hash tx.rlpHash # we only interested in stack, disable all other elems diff --git a/hive_integration/nodocker/engine/test_env.nim b/hive_integration/nodocker/engine/test_env.nim index c893ad613..0af07ee0f 100644 --- a/hive_integration/nodocker/engine/test_env.nim +++ b/hive_integration/nodocker/engine/test_env.nim @@ -116,18 +116,20 @@ func numEngines*(env: TestEnv): int = func accounts*(env: TestEnv, idx: int): TestAccount = env.sender.getAccount(idx) -proc makeTx*(env: TestEnv, tc: BaseTx, nonce: AccountNonce): Transaction = +proc makeTx*( + env: TestEnv, tc: BaseTx, nonce: AccountNonce): PooledTransaction = env.sender.makeTx(tc, nonce) -proc makeTx*(env: TestEnv, tc: BigInitcodeTx, nonce: AccountNonce): Transaction = +proc makeTx*( + env: TestEnv, tc: BigInitcodeTx, nonce: AccountNonce): PooledTransaction = env.sender.makeTx(tc, nonce) -proc makeTxs*(env: TestEnv, tc: BaseTx, num: int): seq[Transaction] = - result = newSeqOfCap[Transaction](num) +proc makeTxs*(env: TestEnv, tc: BaseTx, num: int): seq[PooledTransaction] = + result = newSeqOfCap[PooledTransaction](num) for _ in 0.. expectedVersion: raise unsupportedFork("getPayload" & $expectedVersion & - " expect ExecutionPayload" & $expectedVersion & - " but get ExecutionPayload" & $version) + " expect ExecutionPayload" & $expectedVersion & + " but get ExecutionPayload" & $version) + if blobsBundle.isSome: + raise unsupportedFork("getPayload" & $expectedVersion & + " contains unsupported BlobsBundleV1") GetPayloadV2Response( executionPayload: payloadGeneric.V1V2, @@ -46,38 +50,25 @@ proc getPayloadV3*(ben: BeaconEngineRef, id: PayloadID): GetPayloadV3Response = var payloadGeneric: ExecutionPayload var blockValue: UInt256 - if not ben.get(id, blockValue, payloadGeneric): + var blobsBundle: Option[BlobsBundleV1] + if not ben.get(id, blockValue, payloadGeneric, blobsBundle): raise unknownPayload("Unknown payload") let version = payloadGeneric.version if version != Version.V3: raise unsupportedFork("getPayloadV3 expect ExecutionPayloadV3 but get ExecutionPayload" & $version) + if blobsBundle.isNone: + raise unsupportedFork("getPayloadV3 is missing BlobsBundleV1") let payload = payloadGeneric.V3 let com = ben.com if not com.isCancunOrLater(ethTime payload.timestamp): raise unsupportedFork("payload timestamp is less than Cancun activation") - var - blobsBundle: BlobsBundleV1 - - try: - for ttx in payload.transactions: - let tx = rlp.decode(distinctBase(ttx), Transaction) - if tx.networkPayload.isNil.not: - for blob in tx.networkPayload.blobs: - blobsBundle.blobs.add Web3Blob(blob) - for p in tx.networkPayload.proofs: - blobsBundle.proofs.add Web3KZGProof(p) - for k in tx.networkPayload.commitments: - blobsBundle.commitments.add Web3KZGCommitment(k) - except RlpError: - doAssert(false, "found TypedTransaction that RLP failed to decode") - GetPayloadV3Response( executionPayload: payload, blockValue: blockValue, - blobsBundle: blobsBundle, + blobsBundle: blobsBundle.get, shouldOverrideBuilder: false ) @@ -87,37 +78,24 @@ proc getPayloadV4*(ben: BeaconEngineRef, id: PayloadID): GetPayloadV4Response = var payloadGeneric: ExecutionPayload var blockValue: UInt256 - if not ben.get(id, blockValue, payloadGeneric): + var blobsBundle: Option[BlobsBundleV1] + if not ben.get(id, blockValue, payloadGeneric, blobsBundle): raise unknownPayload("Unknown payload") let version = payloadGeneric.version if version != Version.V4: raise unsupportedFork("getPayloadV4 expect ExecutionPayloadV4 but get ExecutionPayload" & $version) + if blobsBundle.isNone: + raise unsupportedFork("getPayloadV4 is missing BlobsBundleV1") let payload = payloadGeneric.V4 let com = ben.com if not com.isPragueOrLater(ethTime payload.timestamp): raise unsupportedFork("payload timestamp is less than Prague activation") - var - blobsBundle: BlobsBundleV1 - - try: - for ttx in payload.transactions: - let tx = rlp.decode(distinctBase(ttx), Transaction) - if tx.networkPayload.isNil.not: - for blob in tx.networkPayload.blobs: - blobsBundle.blobs.add Web3Blob(blob) - for p in tx.networkPayload.proofs: - blobsBundle.proofs.add Web3KZGProof(p) - for k in tx.networkPayload.commitments: - blobsBundle.commitments.add Web3KZGCommitment(k) - except RlpError: - doAssert(false, "found TypedTransaction that RLP failed to decode") - GetPayloadV4Response( executionPayload: payload, blockValue: blockValue, - blobsBundle: blobsBundle, + blobsBundle: blobsBundle.get, shouldOverrideBuilder: false ) diff --git a/nimbus/beacon/api_handler/api_newpayload.nim b/nimbus/beacon/api_handler/api_newpayload.nim index c3045c704..200b6c485 100644 --- a/nimbus/beacon/api_handler/api_newpayload.nim +++ b/nimbus/beacon/api_handler/api_newpayload.nim @@ -118,20 +118,20 @@ proc newPayload*(ben: BeaconEngineRef, validatePayload(apiVersion, version, payload) validateVersion(com, timestamp, version, apiVersion) - - var header = blockHeader(payload, removeBlobs = true, beaconRoot = ethHash beaconRoot) - + + var header = blockHeader(payload, beaconRoot = ethHash beaconRoot) + if apiVersion >= Version.V3: if versionedHashes.isNone: raise invalidParams("newPayload" & $apiVersion & " expect blobVersionedHashes but got none") if not validateVersionedHashed(payload, versionedHashes.get): return invalidStatus(header.parentHash, "invalid blob versionedHashes") - + let blockHash = ethHash payload.blockHash header.validateBlockHash(blockHash, version).isOkOr: return error - + # If we already have the block locally, ignore the entire execution and just # return a fake success. if db.getBlockHeader(blockHash, header): @@ -195,7 +195,7 @@ proc newPayload*(ben: BeaconEngineRef, trace "Inserting block without sethead", hash = blockHash, number = header.blockNumber - let body = blockBody(payload, removeBlobs = true) + let body = blockBody(payload) let vres = ben.chain.insertBlockWithoutSetHead(header, body) if vres != ValidationResult.OK: let blockHash = latestValidHash(db, parent, ttd) diff --git a/nimbus/beacon/beacon_engine.nim b/nimbus/beacon/beacon_engine.nim index 2f445436f..74bf97097 100644 --- a/nimbus/beacon/beacon_engine.nim +++ b/nimbus/beacon/beacon_engine.nim @@ -8,6 +8,7 @@ # those terms. import + std/sequtils, ./web3_eth_conv, ./payload_conv, web3/execution_types, @@ -80,12 +81,22 @@ proc put*(ben: BeaconEngineRef, ben.queue.put(hash, header) proc put*(ben: BeaconEngineRef, id: PayloadID, - blockValue: UInt256, payload: ExecutionPayload) = - ben.queue.put(id, blockValue, payload) + blockValue: UInt256, payload: ExecutionPayload, + blobsBundle: Option[BlobsBundleV1]) = + ben.queue.put(id, blockValue, payload, blobsBundle) proc put*(ben: BeaconEngineRef, id: PayloadID, - blockValue: UInt256, payload: SomeExecutionPayload) = - ben.queue.put(id, blockValue, payload) + blockValue: UInt256, payload: SomeExecutionPayload, + blobsBundle: Option[BlobsBundleV1]) = + doAssert blobsBundle.isNone == (payload is + ExecutionPayloadV1 | ExecutionPayloadV2) + ben.queue.put(id, blockValue, payload, blobsBundle) + +proc put*(ben: BeaconEngineRef, id: PayloadID, + blockValue: UInt256, + payload: ExecutionPayloadV1 | ExecutionPayloadV2) = + ben.queue.put( + id, blockValue, payload, blobsBundle = options.none(BlobsBundleV1)) # ------------------------------------------------------------------------------ # Public functions, getters @@ -115,8 +126,9 @@ proc get*(ben: BeaconEngineRef, hash: common.Hash256, proc get*(ben: BeaconEngineRef, id: PayloadID, blockValue: var UInt256, - payload: var ExecutionPayload): bool = - ben.queue.get(id, blockValue, payload) + payload: var ExecutionPayload, + blobsBundle: var Option[BlobsBundleV1]): bool = + ben.queue.get(id, blockValue, payload, blobsBundle) proc get*(ben: BeaconEngineRef, id: PayloadID, blockValue: var UInt256, @@ -130,8 +142,9 @@ proc get*(ben: BeaconEngineRef, id: PayloadID, proc get*(ben: BeaconEngineRef, id: PayloadID, blockValue: var UInt256, - payload: var ExecutionPayloadV3): bool = - ben.queue.get(id, blockValue, payload) + payload: var ExecutionPayloadV3, + blobsBundle: var BlobsBundleV1): bool = + ben.queue.get(id, blockValue, payload, blobsBundle) proc get*(ben: BeaconEngineRef, id: PayloadID, blockValue: var UInt256, @@ -142,9 +155,13 @@ proc get*(ben: BeaconEngineRef, id: PayloadID, # Public functions # ------------------------------------------------------------------------------ +type ExecutionPayloadAndBlobsBundle* = object + executionPayload*: ExecutionPayload + blobsBundle*: Option[BlobsBundleV1] + proc generatePayload*(ben: BeaconEngineRef, attrs: PayloadAttributes): - Result[ExecutionPayload, string] = + Result[ExecutionPayloadAndBlobsBundle, string] = wrapException: let xp = ben.txPool @@ -168,12 +185,22 @@ proc generatePayload*(ben: BeaconEngineRef, if pos.timestamp <= headBlock.timestamp: return err "timestamp must be strictly later than parent" - # someBaseFee = true: make sure blk.header + # someBaseFee = true: make sure bundle.blk.header # have the same blockHash with generated payload - let blk = xp.assembleBlock(someBaseFee = true).valueOr: + let bundle = xp.assembleBlock(someBaseFee = true).valueOr: return err(error) - if blk.header.extraData.len > 32: + if bundle.blk.header.extraData.len > 32: return err "extraData length should not exceed 32 bytes" - ok(executionPayload(blk)) + var blobsBundle: Option[BlobsBundleV1] + if bundle.blobsBundle.isSome: + template blobData: untyped = bundle.blobsBundle.get + blobsBundle = options.some BlobsBundleV1( + commitments: blobData.commitments.mapIt it.Web3KZGCommitment, + proofs: blobData.proofs.mapIt it.Web3KZGProof, + blobs: blobData.blobs.mapIt it.Web3Blob) + + ok ExecutionPayloadAndBlobsBundle( + executionPayload: executionPayload(bundle.blk), + blobsBundle: blobsBundle) diff --git a/nimbus/beacon/payload_conv.nim b/nimbus/beacon/payload_conv.nim index e0a653184..379fd772c 100644 --- a/nimbus/beacon/payload_conv.nim +++ b/nimbus/beacon/payload_conv.nim @@ -28,10 +28,10 @@ func wdRoot(x: Option[seq[WithdrawalV1]]): Option[common.Hash256] if x.isNone: none(common.Hash256) else: some(wdRoot x.get) -func txRoot(list: openArray[Web3Tx], removeBlobs: bool): common.Hash256 +func txRoot(list: openArray[Web3Tx]): common.Hash256 {.gcsafe, raises:[RlpError].} = {.noSideEffect.}: - calcTxRoot(ethTxs(list, removeBlobs)) + calcTxRoot(ethTxs(list)) # ------------------------------------------------------------------------------ # Public functions @@ -80,15 +80,14 @@ func executionPayloadV1V2*(blk: EthBlock): ExecutionPayloadV1OrV2 = ) func blockHeader*(p: ExecutionPayload, - removeBlobs: bool, beaconRoot: Option[common.Hash256]): - common.BlockHeader {.gcsafe, raises:[CatchableError].} = + common.BlockHeader {.gcsafe, raises:[CatchableError].} = common.BlockHeader( parentHash : ethHash p.parentHash, ommersHash : EMPTY_UNCLE_HASH, coinbase : ethAddr p.feeRecipient, stateRoot : ethHash p.stateRoot, - txRoot : txRoot(p.transactions, removeBlobs), + txRoot : txRoot p.transactions, receiptRoot : ethHash p.receiptsRoot, bloom : ethBloom p.logsBloom, difficulty : 0.u256, @@ -106,21 +105,20 @@ func blockHeader*(p: ExecutionPayload, parentBeaconBlockRoot: beaconRoot ) -func blockBody*(p: ExecutionPayload, removeBlobs: bool): - common.BlockBody {.gcsafe, raises:[RlpError].} = +func blockBody*(p: ExecutionPayload): + common.BlockBody {.gcsafe, raises:[RlpError].} = common.BlockBody( uncles : @[], - transactions: ethTxs(p.transactions, removeBlobs), + transactions: ethTxs p.transactions, withdrawals : ethWithdrawals p.withdrawals, ) func ethBlock*(p: ExecutionPayload, - removeBlobs: bool, beaconRoot: Option[common.Hash256]): common.EthBlock {.gcsafe, raises:[CatchableError].} = - common.Ethblock( - header : blockHeader(p, removeBlobs, beaconRoot), + common.EthBlock( + header : blockHeader(p, beaconRoot), uncles : @[], - txs : ethTxs(p.transactions, removeBlobs), + txs : ethTxs p.transactions, withdrawals: ethWithdrawals p.withdrawals, ) diff --git a/nimbus/beacon/payload_queue.nim b/nimbus/beacon/payload_queue.nim index 5a96bba0c..07bf1cbd4 100644 --- a/nimbus/beacon/payload_queue.nim +++ b/nimbus/beacon/payload_queue.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2022-2023 Status Research & Development GmbH +# Copyright (c) 2022-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) # * MIT license ([LICENSE-MIT](LICENSE-MIT)) @@ -35,6 +35,7 @@ type id: PayloadID payload: ExecutionPayload blockValue: UInt256 + blobsBundle: Option[BlobsBundleV1] HeaderItem = object hash: common.Hash256 @@ -71,13 +72,22 @@ proc put*(api: var PayloadQueue, api.headerQueue.put(HeaderItem(hash: hash, header: header)) proc put*(api: var PayloadQueue, id: PayloadID, - blockValue: UInt256, payload: ExecutionPayload) = + blockValue: UInt256, payload: ExecutionPayload, + blobsBundle: Option[BlobsBundleV1]) = api.payloadQueue.put(PayloadItem(id: id, - payload: payload, blockValue: blockValue)) + payload: payload, blockValue: blockValue, blobsBundle: blobsBundle)) proc put*(api: var PayloadQueue, id: PayloadID, - blockValue: UInt256, payload: SomeExecutionPayload) = - api.put(id, blockValue, payload.executionPayload) + blockValue: UInt256, payload: SomeExecutionPayload, + blobsBundle: Option[BlobsBundleV1]) = + doAssert blobsBundle.isNone == (payload is + ExecutionPayloadV1 | ExecutionPayloadV2) + api.put(id, blockValue, payload.executionPayload, blobsBundle = blobsBundle) + +proc put*(api: var PayloadQueue, id: PayloadID, + blockValue: UInt256, + payload: ExecutionPayloadV1 | ExecutionPayloadV2) = + api.put(id, blockValue, payload, blobsBundle = options.none(BlobsBundleV1)) # ------------------------------------------------------------------------------ # Public functions, getters @@ -93,46 +103,66 @@ proc get*(api: PayloadQueue, hash: common.Hash256, proc get*(api: PayloadQueue, id: PayloadID, blockValue: var UInt256, - payload: var ExecutionPayload): bool = + payload: var ExecutionPayload, + blobsBundle: var Option[BlobsBundleV1]): bool = for x in api.payloadQueue: if x.id == id: payload = x.payload blockValue = x.blockValue + blobsBundle = x.blobsBundle return true false proc get*(api: PayloadQueue, id: PayloadID, blockValue: var UInt256, payload: var ExecutionPayloadV1): bool = - var p: ExecutionPayload - let found = api.get(id, blockValue, p) - doAssert(p.version == Version.V1) - payload = p.V1 + var + p: ExecutionPayload + blobsBundleOpt: Option[BlobsBundleV1] + let found = api.get(id, blockValue, p, blobsBundleOpt) + if found: + doAssert(p.version == Version.V1) + payload = p.V1 + doAssert(blobsBundleOpt.isNone) return found proc get*(api: PayloadQueue, id: PayloadID, blockValue: var UInt256, payload: var ExecutionPayloadV2): bool = - var p: ExecutionPayload - let found = api.get(id, blockValue, p) - doAssert(p.version == Version.V2) - payload = p.V2 + var + p: ExecutionPayload + blobsBundleOpt: Option[BlobsBundleV1] + let found = api.get(id, blockValue, p, blobsBundleOpt) + if found: + doAssert(p.version == Version.V2) + payload = p.V2 + doAssert(blobsBundleOpt.isNone) return found proc get*(api: PayloadQueue, id: PayloadID, blockValue: var UInt256, - payload: var ExecutionPayloadV3): bool = - var p: ExecutionPayload - let found = api.get(id, blockValue, p) - doAssert(p.version == Version.V3) - payload = p.V3 + payload: var ExecutionPayloadV3, + blobsBundle: var BlobsBundleV1): bool = + var + p: ExecutionPayload + blobsBundleOpt: Option[BlobsBundleV1] + let found = api.get(id, blockValue, p, blobsBundleOpt) + if found: + doAssert(p.version == Version.V3) + payload = p.V3 + doAssert(blobsBundleOpt.isSome) + blobsBundle = blobsBundleOpt.unsafeGet return found proc get*(api: PayloadQueue, id: PayloadID, blockValue: var UInt256, payload: var ExecutionPayloadV1OrV2): bool = - var p: ExecutionPayload - let found = api.get(id, blockValue, p) - doAssert(p.version in {Version.V1, Version.V2}) - payload = p.V1V2 + var + p: ExecutionPayload + blobsBundleOpt: Option[BlobsBundleV1] + let found = api.get(id, blockValue, p, blobsBundleOpt) + if found: + doAssert(p.version in {Version.V1, Version.V2}) + payload = p.V1V2 + doAssert(blobsBundleOpt.isNone) return found diff --git a/nimbus/beacon/web3_eth_conv.nim b/nimbus/beacon/web3_eth_conv.nim index 49af720b1..f231b332b 100644 --- a/nimbus/beacon/web3_eth_conv.nim +++ b/nimbus/beacon/web3_eth_conv.nim @@ -151,15 +151,11 @@ func ethWithdrawals*(x: Option[seq[WithdrawalV1]]): func ethTx*(x: Web3Tx): common.Transaction {.gcsafe, raises:[RlpError].} = result = rlp.decode(distinctBase x, common.Transaction) -func ethTxs*(list: openArray[Web3Tx], removeBlobs = false): +func ethTxs*(list: openArray[Web3Tx]): seq[common.Transaction] {.gcsafe, raises:[RlpError].} = result = newSeqOfCap[common.Transaction](list.len) - if removeBlobs: - for x in list: - result.add ethTx(x).removeNetworkPayload - else: - for x in list: - result.add ethTx(x) + for x in list: + result.add ethTx(x) func storageKeys(list: seq[FixedBytes[32]]): seq[StorageKey] = for x in list: diff --git a/nimbus/core/eip4844.nim b/nimbus/core/eip4844.nim index 03fa56e63..1ab12f8c2 100644 --- a/nimbus/core/eip4844.nim +++ b/nimbus/core/eip4844.nim @@ -167,17 +167,17 @@ func validateEip4844Header*( return ok() -proc validateBlobTransactionWrapper*(tx: Transaction): +proc validateBlobTransactionWrapper*(tx: PooledTransaction): Result[void, string] {.raises: [].} = if tx.networkPayload.isNil: return err("tx wrapper is none") # note: assert blobs are not malformatted - let goodFormatted = tx.versionedHashes.len == + let goodFormatted = tx.tx.versionedHashes.len == tx.networkPayload.commitments.len and - tx.versionedHashes.len == + tx.tx.versionedHashes.len == tx.networkPayload.blobs.len and - tx.versionedHashes.len == + tx.tx.versionedHashes.len == tx.networkPayload.proofs.len if not goodFormatted: @@ -194,12 +194,13 @@ proc validateBlobTransactionWrapper*(tx: Transaction): return err("Failed to verify network payload of a transaction") # Now that all commitments have been verified, check that versionedHashes matches the commitments - for i in 0 ..< tx.versionedHashes.len: + for i in 0 ..< tx.tx.versionedHashes.len: # this additional check also done in tx validation - if tx.versionedHashes[i].data[0] != VERSIONED_HASH_VERSION_KZG: + if tx.tx.versionedHashes[i].data[0] != VERSIONED_HASH_VERSION_KZG: return err("wrong kzg version in versioned hash at index " & $i) - if tx.versionedHashes[i] != kzgToVersionedHash(tx.networkPayload.commitments[i]): + if tx.tx.versionedHashes[i] != + kzgToVersionedHash(tx.networkPayload.commitments[i]): return err("tx versioned hash not match commitments at index " & $i) ok() diff --git a/nimbus/core/sealer.nim b/nimbus/core/sealer.nim index 591e77c94..bbb572540 100644 --- a/nimbus/core/sealer.nim +++ b/nimbus/core/sealer.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018-2023 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) # * MIT license ([LICENSE-MIT](LICENSE-MIT)) @@ -61,8 +61,9 @@ proc validateSealer*(conf: NimbusConf, ctx: EthContext, chain: ChainRef): Result proc generateBlock(engine: SealingEngineRef, outBlock: var EthBlock): Result[void, string] = - outBlock = engine.txPool.assembleBlock().valueOr: + let bundle = engine.txPool.assembleBlock().valueOr: return err(error) + outBlock = bundle.blk if engine.chain.com.consensus == ConsensusType.POS: # Stop the block generator if we reach TTD diff --git a/nimbus/core/tx_pool.nim b/nimbus/core/tx_pool.nim index ed5ff0eac..b4f176505 100644 --- a/nimbus/core/tx_pool.nim +++ b/nimbus/core/tx_pool.nim @@ -423,7 +423,7 @@ ## import - std/[sequtils, tables], + std/[options, sequtils, tables], ./tx_pool/[tx_chain, tx_desc, tx_info, tx_item], ./tx_pool/tx_tabs, ./tx_pool/tx_tasks/[ @@ -517,7 +517,7 @@ proc new*(T: type TxPoolRef; com: CommonRef; miner: EthAddress): T # core/tx_pool.go(848): func (pool *TxPool) AddLocals(txs [].. # core/tx_pool.go(864): func (pool *TxPool) AddRemotes(txs [].. -proc add*(xp: TxPoolRef; txs: openArray[Transaction]; info = "") +proc add*(xp: TxPoolRef; txs: openArray[PooledTransaction]; info = "") {.gcsafe,raises: [CatchableError].} = ## Add a list of transactions to be processed and added to the buckets ## database. It is OK pass an empty list in which case some maintenance @@ -533,7 +533,7 @@ proc add*(xp: TxPoolRef; txs: openArray[Transaction]; info = "") # core/tx_pool.go(854): func (pool *TxPool) AddLocals(txs [].. # core/tx_pool.go(883): func (pool *TxPool) AddRemotes(txs [].. -proc add*(xp: TxPoolRef; tx: Transaction; info = "") +proc add*(xp: TxPoolRef; tx: PooledTransaction; info = "") {.gcsafe,raises: [CatchableError].} = ## Variant of `add()` for a single transaction. xp.add(@[tx], info) @@ -607,8 +607,14 @@ proc dirtyBuckets*(xp: TxPoolRef): bool = ## flag is also set. xp.pDirtyBuckets -proc assembleBlock*(xp: TxPoolRef, someBaseFee: bool = false): Result[EthBlock, string] - {.gcsafe,raises: [CatchableError].} = +type EthBlockAndBlobsBundle* = object + blk*: EthBlock + blobsBundle*: Option[BlobsBundle] + +proc assembleBlock*( + xp: TxPoolRef, + someBaseFee: bool = false +): Result[EthBlockAndBlobsBundle, string] {.gcsafe,raises: [CatchableError].} = ## Getter, retrieves a packed block ready for mining and signing depending ## on the internally cached block chain head, the txs in the pool and some ## tuning parameters. The following block header fields are left @@ -627,19 +633,41 @@ proc assembleBlock*(xp: TxPoolRef, someBaseFee: bool = false): Result[EthBlock, var blk = EthBlock( header: xp.chain.getHeader # uses updated vmState ) + var blobsBundle: BlobsBundle - for (_,nonceList) in xp.txDB.packingOrderAccounts(txItemPacked): - blk.txs.add toSeq(nonceList.incNonce).mapIt(it.tx) + for _, nonceList in xp.txDB.packingOrderAccounts(txItemPacked): + for item in nonceList.incNonce: + let tx = item.pooledTx + blk.txs.add tx.tx + if tx.networkPayload != nil: + for k in tx.networkPayload.commitments: + blobsBundle.commitments.add k + for p in tx.networkPayload.proofs: + blobsBundle.proofs.add p + for blob in tx.networkPayload.blobs: + blobsBundle.blobs.add blob let com = xp.chain.com if com.forkGTE(Shanghai): blk.withdrawals = some(com.pos.withdrawals) + if not com.forkGTE(Cancun) and blobsBundle.commitments.len > 0: + return err("PooledTransaction contains blobs prior to Cancun") + let blobsBundleOpt = + if com.forkGTE(Cancun): + doAssert blobsBundle.commitments.len == blobsBundle.blobs.len + doAssert blobsBundle.proofs.len == blobsBundle.blobs.len + options.some blobsBundle + else: + options.none BlobsBundle + if someBaseFee: # make sure baseFee always has something blk.header.fee = some(blk.header.fee.get(0.u256)) - ok(blk) + ok EthBlockAndBlobsBundle( + blk: blk, + blobsBundle: blobsBundleOpt) proc gasCumulative*(xp: TxPoolRef): GasInt = ## Getter, retrieves the gas that will be burned in the block after @@ -856,7 +884,7 @@ proc accountRanks*(xp: TxPoolRef): TxTabsLocality = xp.txDB.locality proc addRemote*(xp: TxPoolRef; - tx: Transaction; force = false): Result[void,TxInfo] + tx: PooledTransaction; force = false): Result[void,TxInfo] {.gcsafe,raises: [CatchableError].} = ## Adds the argument transaction `tx` to the buckets database. ## @@ -890,7 +918,7 @@ proc addRemote*(xp: TxPoolRef; ok() proc addLocal*(xp: TxPoolRef; - tx: Transaction; force = false): Result[void,TxInfo] + tx: PooledTransaction; force = false): Result[void,TxInfo] {.gcsafe,raises: [CatchableError].} = ## Adds the argument transaction `tx` to the buckets database. ## diff --git a/nimbus/core/tx_pool/tx_item.nim b/nimbus/core/tx_pool/tx_item.nim index f309046c1..2d1cb2e01 100644 --- a/nimbus/core/tx_pool/tx_item.nim +++ b/nimbus/core/tx_pool/tx_item.nim @@ -42,7 +42,7 @@ type TxItemRef* = ref object of RootObj ##\ ## Data container with transaction and meta data. Entries are *read-only*\ ## by default, for some there is a setter available. - tx: Transaction ## Transaction data + tx: PooledTransaction ## Transaction data itemID: Hash256 ## Transaction hash timeStamp: Time ## Time when added sender: EthAddress ## Sender account address @@ -112,10 +112,10 @@ proc init*(item: TxItemRef; status: TxItemStatus; info: string) = item.timeStamp = utcTime() item.reject = txInfoOk -proc new*(T: type TxItemRef; tx: Transaction; itemID: Hash256; +proc new*(T: type TxItemRef; tx: PooledTransaction; itemID: Hash256; status: TxItemStatus; info: string): Result[T,void] {.gcsafe,raises: [].} = ## Create item descriptor. - let rc = tx.ecRecover + let rc = tx.tx.ecRecover if rc.isErr: return err() ok(T(itemID: itemID, @@ -125,7 +125,7 @@ proc new*(T: type TxItemRef; tx: Transaction; itemID: Hash256; info: info, status: status)) -proc new*(T: type TxItemRef; tx: Transaction; +proc new*(T: type TxItemRef; tx: PooledTransaction; reject: TxInfo; status: TxItemStatus; info: string): T {.gcsafe,raises: [].} = ## Create incomplete item descriptor, so meta-data can be stored (e.g. ## for holding in the waste basket to be investigated later.) @@ -150,6 +150,10 @@ proc itemID*(tx: Transaction): Hash256 = ## Getter, transaction ID tx.rlpHash +proc itemID*(tx: PooledTransaction): Hash256 = + ## Getter, transaction ID + tx.tx.rlpHash + # core/types/transaction.go(297): func (tx *Transaction) Cost() *big.Int { proc cost*(tx: Transaction): UInt256 = ## Getter (go/ref compat): gas * gasPrice + value. @@ -210,10 +214,14 @@ proc timeStamp*(item: TxItemRef): Time = ## Getter item.timeStamp -proc tx*(item: TxItemRef): Transaction = +proc pooledTx*(item: TxItemRef): PooledTransaction = ## Getter item.tx +proc tx*(item: TxItemRef): Transaction = + ## Getter + item.tx.tx + func rejectInfo*(item: TxItemRef): string = ## Getter result = $item.reject diff --git a/nimbus/core/tx_pool/tx_tabs.nim b/nimbus/core/tx_pool/tx_tabs.nim index 2f7fdef4a..2f785cb0b 100644 --- a/nimbus/core/tx_pool/tx_tabs.nim +++ b/nimbus/core/tx_pool/tx_tabs.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -138,7 +138,7 @@ proc new*(T: type TxTabsRef): T {.gcsafe,raises: [].} = proc insert*( xp: TxTabsRef; - tx: var Transaction; + tx: var PooledTransaction; status = txItemPending; info = ""): Result[void,TxInfo] {.gcsafe,raises: [CatchableError].} = @@ -221,7 +221,7 @@ proc dispose*(xp: TxTabsRef; item: TxItemRef; reason: TxInfo): bool xp.byRejects[item.itemID] = item return true -proc reject*(xp: TxTabsRef; tx: var Transaction; +proc reject*(xp: TxTabsRef; tx: var PooledTransaction; reason: TxInfo; status = txItemPending; info = "") = ## Similar to dispose but for a tx without the item wrapper, the function ## imports the tx into the waste basket (e.g. after it could not @@ -239,7 +239,7 @@ proc reject*(xp: TxTabsRef; item: TxItemRef; reason: TxInfo) = item.reject = reason xp.byRejects[item.itemID] = item -proc reject*(xp: TxTabsRef; tx: Transaction; +proc reject*(xp: TxTabsRef; tx: PooledTransaction; reason: TxInfo; status = txItemPending; info = "") = ## Variant of `reject()` var ty = tx diff --git a/nimbus/core/tx_pool/tx_tasks/tx_add.nim b/nimbus/core/tx_pool/tx_tasks/tx_add.nim index 9d8297406..01e95f087 100644 --- a/nimbus/core/tx_pool/tx_tasks/tx_add.nim +++ b/nimbus/core/tx_pool/tx_tasks/tx_add.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -160,7 +160,7 @@ proc addTx*(xp: TxPoolRef; item: TxItemRef): bool # core/tx_pool.go(883): func (pool *TxPool) AddRemotes(txs [].. # core/tx_pool.go(889): func (pool *TxPool) addTxs(txs []*types.Transaction, .. proc addTxs*(xp: TxPoolRef; - txs: openArray[Transaction]; info = ""): TxAddStats + txs: openArray[PooledTransaction]; info = ""): TxAddStats {.discardable,gcsafe,raises: [CatchableError].} = ## Add a list of transactions. The list is sorted after nonces and txs are ## tested and stored into either of the `pending` or `staged` buckets, or @@ -181,7 +181,7 @@ proc addTxs*(xp: TxPoolRef; for tx in txs.items: var reason: TxInfo - if tx.txType == TxEip4844: + if tx.tx.txType == TxEip4844: let res = tx.validateBlobTransactionWrapper() if res.isErr: # move item to waste basket diff --git a/nimbus/core/tx_pool/tx_tasks/tx_classify.nim b/nimbus/core/tx_pool/tx_tasks/tx_classify.nim index 49a1a77a6..dedb6a240 100644 --- a/nimbus/core/tx_pool/tx_tasks/tx_classify.nim +++ b/nimbus/core/tx_pool/tx_tasks/tx_classify.nim @@ -38,7 +38,7 @@ logScope: proc checkTxBasic(xp: TxPoolRef; item: TxItemRef): bool = let res = validateTxBasic( - item.tx.removeNetworkPayload, + item.tx, xp.chain.nextFork, # A new transaction of the next fork may be # coming before the fork activated @@ -234,7 +234,8 @@ proc classifyValidatePacked*(xp: TxPoolRef; tx = item.tx.eip1559TxNormalization(xp.chain.baseFee.GasInt) excessBlobGas = calcExcessBlobGas(vmState.parent) - roDB.validateTransaction(tx.removeNetworkPayload, item.sender, gasLimit, baseFee, excessBlobGas, fork).isOk + roDB.validateTransaction( + tx, item.sender, gasLimit, baseFee, excessBlobGas, fork).isOk proc classifyPacked*(xp: TxPoolRef; gasBurned, moreBurned: GasInt): bool = ## Classifier for *packing* (i.e. adding up `gasUsed` values after executing diff --git a/nimbus/core/tx_pool/tx_tasks/tx_head.nim b/nimbus/core/tx_pool/tx_tasks/tx_head.nim index 063e04351..aeb53bca6 100644 --- a/nimbus/core/tx_pool/tx_tasks/tx_head.nim +++ b/nimbus/core/tx_pool/tx_tasks/tx_head.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -31,7 +31,7 @@ type ## Diff data, txs changes that apply after changing the head\ ## insertion point of the block chain - addTxs*: KeyedQueue[Hash256,Transaction] ##\ + addTxs*: KeyedQueue[Hash256, PooledTransaction] ##\ ## txs to add; using a queue makes it more intuive to delete ## items while travesing the queue in a loop. @@ -50,7 +50,13 @@ proc insert(xp: TxPoolRef; kq: TxHeadDiffRef; blockHash: Hash256) {.gcsafe,raises: [CatchableError].} = let db = xp.chain.com.db for tx in db.getBlockBody(blockHash).transactions: - kq.addTxs[tx.itemID] = tx + if tx.versionedHashes.len > 0: + # EIP-4844 blobs are not persisted and cannot be re-broadcasted. + # Note that it is also not possible to crete a cache in all cases, + # as we may have never seen the actual blob sidecar while syncing. + # Only the consensus layer persists the blob sidecar. + continue + kq.addTxs[tx.itemID] = PooledTransaction(tx: tx) proc remove(xp: TxPoolRef; kq: TxHeadDiffRef; blockHash: Hash256) {.gcsafe,raises: [CatchableError].} = diff --git a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim index 25df1d0aa..c021d0387 100644 --- a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim +++ b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim @@ -141,7 +141,7 @@ proc runTxCommit(pst: TxPackerStateRef; item: TxItemRef; gasBurned: GasInt) pst.blobGasUsed += item.tx.getTotalBlobGas # Update txRoot - pst.tr.put(rlp.encode(inx), rlp.encode(item.tx.removeNetworkPayload)) + pst.tr.put(rlp.encode(inx), rlp.encode(item.tx)) # Add the item to the `packed` bucket. This implicitely increases the # receipts index `inx` at the next visit of this function. diff --git a/nimbus/core/tx_pool/tx_tasks/tx_recover.nim b/nimbus/core/tx_pool/tx_tasks/tx_recover.nim index 243930190..ab48c6444 100644 --- a/nimbus/core/tx_pool/tx_tasks/tx_recover.nim +++ b/nimbus/core/tx_pool/tx_tasks/tx_recover.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2018 Status Research & Development GmbH +# Copyright (c) 2018-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -35,7 +35,7 @@ let # Public functions # ------------------------------------------------------------------------------ -proc recoverItem*(xp: TxPoolRef; tx: Transaction; status = txItemPending; +proc recoverItem*(xp: TxPoolRef; tx: PooledTransaction; status = txItemPending; info = ""; acceptExisting = false): Result[TxItemRef,TxInfo] = ## Recover item from waste basket or create new. It is an error if the item ## is in the buckets database, already. diff --git a/nimbus/core/validate.nim b/nimbus/core/validate.nim index b45ca5b87..5fba80dcf 100644 --- a/nimbus/core/validate.nim +++ b/nimbus/core/validate.nim @@ -267,9 +267,6 @@ proc validateTxBasic*( "index=$1, len=$2" % [$i, $acl.storageKeys.len]) if tx.txType >= TxEip4844: - if tx.networkPayload.isNil.not: - return err("invalid tx: network payload should not appear in block validation") - if tx.to.isNone: return err("invalid tx: destination must be not empty") diff --git a/nimbus/db/core_db/core_apps_newapi.nim b/nimbus/db/core_db/core_apps_newapi.nim index 502099ed6..209646f28 100644 --- a/nimbus/db/core_db/core_apps_newapi.nim +++ b/nimbus/db/core_db/core_apps_newapi.nim @@ -558,8 +558,8 @@ proc persistTransactions*( for idx, tx in transactions: let encodedKey = rlp.encode(idx) - encodedTx = rlp.encode(tx.removeNetworkPayload) - txHash = rlpHash(tx) # beware EIP-4844 + encodedTx = rlp.encode(tx) + txHash = rlpHash(tx) blockKey = transactionHashToBlockKey(txHash) txKey: TransactionKey = (blockNumber, idx) mpt.merge(encodedKey, encodedTx).isOkOr: diff --git a/nimbus/graphql/ethapi.nim b/nimbus/graphql/ethapi.nim index affe672aa..fd5d77ed6 100644 --- a/nimbus/graphql/ethapi.nim +++ b/nimbus/graphql/ethapi.nim @@ -1364,8 +1364,8 @@ proc sendRawTransaction(ud: RootRef, params: Args, parent: Node): RespResult {.a let ctx = GraphqlContextRef(ud) try: let data = hexToSeqByte(params[0].val.stringVal) - let tx = decodeTx(data) # we want to know if it is a valid tx blob - let txHash = rlpHash(tx) # beware EIP-4844 + let tx = decodePooledTx(data) # we want to know if it is a valid tx blob + let txHash = rlpHash(tx) ctx.txPool.add(tx) diff --git a/nimbus/rpc/p2p.nim b/nimbus/rpc/p2p.nim index ec0d7e64f..ca56e15f4 100644 --- a/nimbus/rpc/p2p.nim +++ b/nimbus/rpc/p2p.nim @@ -10,7 +10,7 @@ {.push raises: [].} import - std/[times, tables, typetraits], + std/[sequtils, times, tables, typetraits], json_rpc/rpcserver, stint, stew/byteutils, json_serialization, web3/conversions, json_serialization/std/options, eth/common/eth_types_json_serialization, @@ -282,8 +282,27 @@ proc setupEthRpc*( tx = unsignedTx(data, chainDB, accDB.getNonce(address) + 1) eip155 = com.isEIP155(com.syncCurrent) signedTx = signTransaction(tx, acc.privateKey, com.chainId, eip155) + networkPayload = + if signedTx.txType == TxEip4844: + if data.blobs.isNone or data.commitments.isNone or data.proofs.isNone: + raise newException(ValueError, "EIP-4844 transaction needs blobs") + if data.blobs.get.len != signedTx.versionedHashes.len: + raise newException(ValueError, "Incorrect number of blobs") + if data.commitments.get.len != signedTx.versionedHashes.len: + raise newException(ValueError, "Incorrect number of commitments") + if data.proofs.get.len != signedTx.versionedHashes.len: + raise newException(ValueError, "Incorrect number of proofs") + NetworkPayload( + blobs: data.blobs.get.mapIt it.NetworkBlob, + commitments: data.commitments.get.mapIt eth_types.KzgCommitment(it), + proofs: data.proofs.get.mapIt eth_types.KzgProof(it)) + else: + if data.blobs.isSome or data.commitments.isSome or data.proofs.isSome: + raise newException(ValueError, "Blobs require EIP-4844 transaction") + nil + pooledTx = PooledTransaction(tx: signedTx, networkPayload: networkPayload) - txPool.add(signedTx) + txPool.add(pooledTx) result = rlpHash(signedTx).w3Hash server.rpc("eth_sendRawTransaction") do(txBytes: seq[byte]) -> Web3Hash: @@ -293,10 +312,10 @@ proc setupEthRpc*( ## Returns the transaction hash, or the zero hash if the transaction is not yet available. ## Note: Use eth_getTransactionReceipt to get the contract address, after the transaction was mined, when you created a contract. let - signedTx = decodeTx(txBytes) - txHash = rlpHash(signedTx) + pooledTx = decodePooledTx(txBytes) + txHash = rlpHash(pooledTx) - txPool.add(signedTx) + txPool.add(pooledTx) let res = txPool.inPoolAndReason(txHash) if res.isErr: raise newException(ValueError, res.error) diff --git a/nimbus/sync/handlers/eth.nim b/nimbus/sync/handlers/eth.nim index c8ec5e22d..bbc83de21 100644 --- a/nimbus/sync/handlers/eth.nim +++ b/nimbus/sync/handlers/eth.nim @@ -442,14 +442,14 @@ method getReceipts*(ctx: EthWireRef, method getPooledTxs*(ctx: EthWireRef, hashes: openArray[Hash256]): - Result[seq[Transaction], string] + Result[seq[PooledTransaction], string] {.gcsafe.} = let txPool = ctx.txPool - var list: seq[Transaction] + var list: seq[PooledTransaction] for txHash in hashes: let res = txPool.getItem(txHash) if res.isOk: - list.add res.value.tx + list.add res.value.pooledTx else: trace "handlers.getPooledTxs: tx not found", txHash ok(list) @@ -522,7 +522,11 @@ method handleAnnouncedTxs*(ctx: EthWireRef, txHashes.add rlpHash(tx) ctx.addToKnownByPeer(txHashes, peer) - ctx.txPool.add(txs) + for tx in txs: + if tx.versionedHashes.len > 0: + # EIP-4844 blobs are not persisted and cannot be broadcasted + continue + ctx.txPool.add PooledTransaction(tx: tx) var newTxHashes = newSeqOfCap[Hash256](txHashes.len) var validTxs = newSeqOfCap[Transaction](txHashes.len) diff --git a/nimbus/sync/protocol/eth/eth_types.nim b/nimbus/sync/protocol/eth/eth_types.nim index 716ddcfd7..ef62c2b4a 100644 --- a/nimbus/sync/protocol/eth/eth_types.nim +++ b/nimbus/sync/protocol/eth/eth_types.nim @@ -65,7 +65,7 @@ method getReceipts*(ctx: EthWireBase, method getPooledTxs*(ctx: EthWireBase, hashes: openArray[Hash256]): - Result[seq[Transaction], string] + Result[seq[PooledTransaction], string] {.base, gcsafe.} = notImplemented("getPooledTxs") diff --git a/nimbus/sync/protocol/eth66.nim b/nimbus/sync/protocol/eth66.nim index f20bec917..015f4c227 100644 --- a/nimbus/sync/protocol/eth66.nim +++ b/nimbus/sync/protocol/eth66.nim @@ -266,7 +266,8 @@ p2pProtocol eth66(version = ethVersion, await response.send(txs.get) # User message 0x0a: PooledTransactions. - proc pooledTransactions(peer: Peer, transactions: openArray[Transaction]) + proc pooledTransactions( + peer: Peer, transactions: openArray[PooledTransaction]) nextId 0x0d diff --git a/nimbus/sync/protocol/eth67.nim b/nimbus/sync/protocol/eth67.nim index 18c6add77..c3b610ab1 100644 --- a/nimbus/sync/protocol/eth67.nim +++ b/nimbus/sync/protocol/eth67.nim @@ -267,7 +267,8 @@ p2pProtocol eth67(version = ethVersion, await response.send(txs.get) # User message 0x0a: PooledTransactions. - proc pooledTransactions(peer: Peer, transactions: openArray[Transaction]) + proc pooledTransactions( + peer: Peer, transactions: openArray[PooledTransaction]) # User message 0x0d: GetNodeData -- removed, was so 66ish # User message 0x0e: NodeData -- removed, was so 66ish diff --git a/nimbus/sync/protocol/eth68.nim b/nimbus/sync/protocol/eth68.nim index 73b7cdac9..488b55b4f 100644 --- a/nimbus/sync/protocol/eth68.nim +++ b/nimbus/sync/protocol/eth68.nim @@ -270,7 +270,8 @@ p2pProtocol eth68(version = ethVersion, await response.send(txs.get) # User message 0x0a: PooledTransactions. - proc pooledTransactions(peer: Peer, transactions: openArray[Transaction]) + proc pooledTransactions( + peer: Peer, transactions: openArray[PooledTransaction]) # User message 0x0d: GetNodeData -- removed, was so 66ish # User message 0x0e: NodeData -- removed, was so 66ish diff --git a/nimbus/transaction.nim b/nimbus/transaction.nim index 68c258770..c57afebcc 100644 --- a/nimbus/transaction.nim +++ b/nimbus/transaction.nim @@ -233,3 +233,9 @@ proc decodeTx*(bytes: openArray[byte]): Transaction = result = rlp.read(Transaction) if rlp.hasData: raise newException(RlpError, "rlp: input contains more than one value") + +proc decodePooledTx*(bytes: openArray[byte]): PooledTransaction = + var rlp = rlpFromBytes(bytes) + result = rlp.read(PooledTransaction) + if rlp.hasData: + raise newException(RlpError, "rlp: input contains more than one value") diff --git a/nimbus/utils/debug.nim b/nimbus/utils/debug.nim index d379557d7..328f81d8a 100644 --- a/nimbus/utils/debug.nim +++ b/nimbus/utils/debug.nim @@ -134,7 +134,12 @@ proc debug*(tx: Transaction): string = result.add "accessList : " & $tx.accessList & "\n" result.add "maxFeePerBlobGas: " & $tx.maxFeePerBlobGas & "\n" result.add "versionedHashes.len: " & $tx.versionedHashes.len & "\n" + result.add "V : " & $tx.V & "\n" + result.add "R : " & $tx.R & "\n" + result.add "S : " & $tx.S & "\n" +proc debug*(tx: PooledTransaction): string = + result.add debug(tx.tx) if tx.networkPayload.isNil: result.add "networkPaylod : nil\n" else: @@ -143,10 +148,6 @@ proc debug*(tx: Transaction): string = result.add " - commitments : " & $tx.networkPayload.commitments.len & "\n" result.add " - proofs : " & $tx.networkPayload.proofs.len & "\n" - result.add "V : " & $tx.V & "\n" - result.add "R : " & $tx.R & "\n" - result.add "S : " & $tx.S & "\n" - proc debugSum*(h: BlockHeader): string = result.add "txRoot : " & $h.txRoot & "\n" result.add "ommersHash : " & $h.ommersHash & "\n" diff --git a/nimbus_verified_proxy/libverifproxy/verifproxy.nim b/nimbus_verified_proxy/libverifproxy/verifproxy.nim index 4bc6ccccd..b30685b14 100644 --- a/nimbus_verified_proxy/libverifproxy/verifproxy.nim +++ b/nimbus_verified_proxy/libverifproxy/verifproxy.nim @@ -28,6 +28,7 @@ proc initLib() = nimGC_setStackBottom(locals) proc runContext(ctx: ptr Context) {.thread.} = + const defaultListenAddress = (static parseIpAddress("0.0.0.0")) let str = $ctx.configJson try: let jsonNode = parseJson(str) @@ -35,7 +36,7 @@ proc runContext(ctx: ptr Context) {.thread.} = let rpcAddr = jsonNode["RpcAddress"].getStr() let myConfig = VerifiedProxyConf( rpcAddress: parseIpAddress(rpcAddr), - listenAddress: defaultListenAddress, + listenAddress: some(defaultListenAddress), eth2Network: some(jsonNode["Eth2Network"].getStr()), trustedBlockRoot: Eth2Digest.fromHex(jsonNode["TrustedBlockRoot"].getStr()), web3Url: parseCmdArg(Web3Url, jsonNode["Web3Url"].getStr()), diff --git a/nimbus_verified_proxy/nimbus_verified_proxy_conf.nim b/nimbus_verified_proxy/nimbus_verified_proxy_conf.nim index 67e9c357a..6ddca6108 100644 --- a/nimbus_verified_proxy/nimbus_verified_proxy_conf.nim +++ b/nimbus_verified_proxy/nimbus_verified_proxy_conf.nim @@ -108,10 +108,8 @@ type VerifiedProxyConf* = object # Config listenAddress* {. desc: "Listening address for the Ethereum LibP2P and Discovery v5 traffic", - defaultValue: defaultListenAddress, - defaultValueDesc: $defaultListenAddressDesc, name: "listen-address" - .}: IpAddress + .}: Option[IpAddress] tcpPort* {. desc: "Listening TCP port for Ethereum LibP2P traffic", diff --git a/nimbus_verified_proxy/rpc/rpc_utils.nim b/nimbus_verified_proxy/rpc/rpc_utils.nim index 698c66f37..6a97d1eec 100644 --- a/nimbus_verified_proxy/rpc/rpc_utils.nim +++ b/nimbus_verified_proxy/rpc/rpc_utils.nim @@ -34,9 +34,7 @@ type ExecutionData* = object transactions*: seq[TypedTransaction] withdrawals*: seq[WithdrawalV1] -proc asExecutionData*( - payload: ExecutionPayloadV1 | ExecutionPayloadV2 | ExecutionPayloadV3 -): ExecutionData = +proc asExecutionData*(payload: SomeExecutionPayload): ExecutionData = when payload is ExecutionPayloadV1: return ExecutionData( parentHash: payload.parentHash, diff --git a/tests/replay/pp.nim b/tests/replay/pp.nim index c2f7f7663..4df3bd03f 100644 --- a/tests/replay/pp.nim +++ b/tests/replay/pp.nim @@ -100,7 +100,6 @@ func pp*(t: Transaction; sep = " "): string = &"accessList=[#{t.accessList.len}]{sep}" & &"maxFeePerBlobGas={t.maxFeePerBlobGas}{sep}" & &"versionedHashes=[#{t.versionedHashes.len}]{sep}" & - &"networkPayload={t.networkPayload.pp}{sep}" & &"V={t.V}{sep}" & &"R={t.R}{sep}" & &"S={t.S}{sep}" diff --git a/tests/test_eip4844.nim b/tests/test_eip4844.nim index 61db3a16a..d1c4b5849 100644 --- a/tests/test_eip4844.nim +++ b/tests/test_eip4844.nim @@ -1,5 +1,5 @@ # Nimbus -# Copyright (c) 2023 Status Research & Development GmbH +# Copyright (c) 2023-2024 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or # http://www.apache.org/licenses/LICENSE-2.0) @@ -16,11 +16,9 @@ import const recipient = hexToByteArray[20]("095e7baea6a6c7c4c2dfeb977efac326af552d87") - zeroG1 = hexToByteArray[48]("0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") source = hexToByteArray[20]("0x0000000000000000000000000000000000000001") storageKey= default(StorageKey) accesses = @[AccessPair(address: source, storageKeys: @[storageKey])] - blob = default(NetworkBlob) abcdef = hexToSeqByte("abcdef") hexKey = "af1a9be9f1a54421cac82943820a0fe0f601bb5f4f6d0bccc81c613f0ce6ae22" senderTop = hexToByteArray[20]("73cf19657412508833f618a15e8251306b3e6ee5") @@ -98,12 +96,7 @@ proc tx6(i: int): Transaction = maxPriorityFee: 42.GasInt, maxFee: 10.GasInt, accessList: accesses, - versionedHashes: @[digest], - networkPayload: NetworkPayload( - commitments: @[zeroG1], - blobs: @[blob], - proofs: @[zeroG1], - ) + versionedHashes: @[digest] ) proc tx7(i: int): Transaction = diff --git a/tests/test_txpool.nim b/tests/test_txpool.nim index 21c2b59ce..0e201c01b 100644 --- a/tests/test_txpool.nim +++ b/tests/test_txpool.nim @@ -287,7 +287,7 @@ proc runTxPoolTests(noisy = true) = # insert some txs for triple in testTxs: - xq.add(triple[1], triple[0].info) + xq.add(PooledTransaction(tx: triple[1]), triple[0].info) check xq.nItems.total == testTxs.len check xq.nItems.disposed == 0 @@ -296,7 +296,7 @@ proc runTxPoolTests(noisy = true) = # re-insert modified transactions for triple in testTxs: - xq.add(triple[2], "alt " & triple[0].info) + xq.add(PooledTransaction(tx: triple[2]), "alt " & triple[0].info) check xq.nItems.total == testTxs.len check xq.nItems.disposed == testTxs.len @@ -505,7 +505,7 @@ proc runTxPoolTests(noisy = true) = check txList.len == xq.nItems.total + xq.nItems.disposed # re-add item - xq.add(thisItem.tx) + xq.add(thisItem.pooledTx) # verify that the pivot item was moved out from the waste basket check not xq.txDB.byRejects.hasKey(thisItem.itemID) @@ -793,7 +793,7 @@ proc runTxPackerTests(noisy = true) = check false return - let blk = r.get + let blk = r.get.blk # Make sure that there are at least two txs on the packed block so # this test does not degenerate. check 1 < xq.chain.receipts.len diff --git a/tests/test_txpool/setup.nim b/tests/test_txpool/setup.nim index 2c0159791..5142c6a3c 100644 --- a/tests/test_txpool/setup.nim +++ b/tests/test_txpool/setup.nim @@ -103,7 +103,7 @@ proc toTxPool*( status = statusInfo[getStatus()] info = &"{txCount} #{num}({chainNo}) {n}/{txs.len} {status}" noisy.showElapsed(&"insert: {info}"): - result[0].add(txs[n], info) + result[0].add(PooledTransaction(tx: txs[n]), info) if loadTxs <= txCount: break @@ -132,11 +132,11 @@ proc toTxPool*( noisy.showElapsed(&"Loading {itList.len} transactions"): for item in itList: if noLocals: - result.add(item.tx, item.info) + result.add(item.pooledTx, item.info) elif localAddr.hasKey(item.sender): - doAssert result.addLocal(item.tx, true).isOk + doAssert result.addLocal(item.pooledTx, true).isOk else: - doAssert result.addRemote(item.tx, true).isOk + doAssert result.addRemote(item.pooledTx, true).isOk doAssert result.nItems.total == itList.len @@ -174,11 +174,11 @@ proc toTxPool*( for n in 0 ..< itList.len: let item = itList[n] if noLocals: - result.add(item.tx, item.info) + result.add(item.pooledTx, item.info) elif localAddr.hasKey(item.sender): - doAssert result.addLocal(item.tx, true).isOk + doAssert result.addLocal(item.pooledTx, true).isOk else: - doAssert result.addRemote(item.tx, true).isOk + doAssert result.addRemote(item.pooledTx, true).isOk if n < 3 or delayAt-3 <= n and n <= delayAt+3 or itList.len-4 < n: let t = result.getItem(item.itemID).value.timeStamp.format(tFmt, utc()) noisy.say &"added item {n} time={t}" diff --git a/tests/test_txpool2.nim b/tests/test_txpool2.nim index b75900ffc..7332a3164 100644 --- a/tests/test_txpool2.nim +++ b/tests/test_txpool2.nim @@ -156,7 +156,7 @@ proc runTxPoolCliqueTest*() = suite "Test TxPool with Clique sealer": test "TxPool addLocal": - let res = xp.addLocal(tx, force = true) + let res = xp.addLocal(PooledTransaction(tx: tx), force = true) check res.isOk if res.isErr: debugEcho res.error @@ -172,7 +172,7 @@ proc runTxPoolCliqueTest*() = check false return - blk = res.get + blk = res.get.blk body = BlockBody( transactions: blk.txs, uncles: blk.uncles @@ -201,7 +201,7 @@ proc runTxPoolCliqueTest*() = check xp.smartHead(blk.header) let tx = env.makeTx(recipient, amount) - let res = xp.addLocal(tx, force = true) + let res = xp.addLocal(PooledTransaction(tx: tx), force = true) check res.isOk if res.isErr: debugEcho res.error @@ -214,7 +214,7 @@ proc runTxPoolCliqueTest*() = check false return - blk = r.get + blk = r.get.blk body = BlockBody( transactions: blk.txs, uncles: blk.uncles @@ -249,7 +249,7 @@ proc runTxPoolPosTest*() = suite "Test TxPool with PoS block": test "TxPool addLocal": - let res = xp.addLocal(tx, force = true) + let res = xp.addLocal(PooledTransaction(tx: tx), force = true) check res.isOk if res.isErr: debugEcho res.error @@ -269,7 +269,7 @@ proc runTxPoolPosTest*() = check false return - blk = r.get + blk = r.get.blk check com.isBlockAfterTtd(blk.header) body = BlockBody( @@ -310,12 +310,12 @@ proc runTxPoolBlobhashTest*() = suite "Test TxPool with blobhash block": test "TxPool addLocal": - let res = xp.addLocal(tx1, force = true) + let res = xp.addLocal(PooledTransaction(tx: tx1), force = true) check res.isOk if res.isErr: debugEcho res.error return - let res2 = xp.addLocal(tx2, force = true) + let res2 = xp.addLocal(PooledTransaction(tx: tx2), force = true) check res2.isOk test "TxPool jobCommit": @@ -332,7 +332,7 @@ proc runTxPoolBlobhashTest*() = check false return - blk = r.get + blk = r.get.blk check com.isBlockAfterTtd(blk.header) body = BlockBody( @@ -366,7 +366,7 @@ proc runTxPoolBlobhashTest*() = xp = env.xp check xp.smartHead(blk.header) - let res = xp.addLocal(tx4, force = true) + let res = xp.addLocal(PooledTransaction(tx: tx4), force = true) check res.isOk if res.isErr: debugEcho res.error @@ -400,7 +400,7 @@ proc runTxHeadDelta*(noisy = true) = let tx = env.makeTx(recipient, amount) # Instead of `add()`, the functions `addRemote()` or `addLocal()` # also would do. - xp.add(tx) + xp.add(PooledTransaction(tx: tx)) noisy.say "***", "txDB", &" n={n}", @@ -418,7 +418,7 @@ proc runTxHeadDelta*(noisy = true) = check false return - let blk = r.get + let blk = r.get.blk check com.isBlockAfterTtd(blk.header) let body = BlockBody( diff --git a/vendor/nim-eth b/vendor/nim-eth index d8209f623..c482b4c5b 160000 --- a/vendor/nim-eth +++ b/vendor/nim-eth @@ -1 +1 @@ -Subproject commit d8209f623f837d14c43a9e3fd464b0e199c5d180 +Subproject commit c482b4c5b658a77cc96b49d4a397aa6d98472ac7 diff --git a/vendor/nim-libp2p b/vendor/nim-libp2p index 7faa0fac2..21cbe3a91 160000 --- a/vendor/nim-libp2p +++ b/vendor/nim-libp2p @@ -1 +1 @@ -Subproject commit 7faa0fac238c2209e7bbbb4755af8f8a8c3834ad +Subproject commit 21cbe3a91a70811522554e89e6a791172cebfef2 diff --git a/vendor/nimbus-eth2 b/vendor/nimbus-eth2 index fc9bc1da3..87605d08a 160000 --- a/vendor/nimbus-eth2 +++ b/vendor/nimbus-eth2 @@ -1 +1 @@ -Subproject commit fc9bc1da3ae7dde04f4591eba302f9e8b20c3924 +Subproject commit 87605d08a7f9cfc3b223bd32143e93a6cdf351ac