From 6b8a7b009eb94bfdac1410f243865984bdd7c4f2 Mon Sep 17 00:00:00 2001 From: jangko Date: Fri, 20 Jan 2023 11:34:31 +0700 Subject: [PATCH] implement EIP-4844: Shard Blobs Transactions --- eth.nimble | 2 +- eth/common/eth_types.nim | 38 ++++- eth/common/eth_types_rlp.nim | 96 +++++++++++-- eth/common/transaction.nim | 25 +++- eth/common/utils.nim | 4 +- eth/p2p/private/p2p_types.nim | 14 +- eth/rlp.nim | 3 + tests/common/all_tests.nim | 14 ++ tests/common/test_eip4844.nim | 236 ++++++++++++++++++++++++++++++++ tests/common/test_eth_types.nim | 2 + 10 files changed, 413 insertions(+), 21 deletions(-) create mode 100644 tests/common/all_tests.nim create mode 100644 tests/common/test_eip4844.nim diff --git a/eth.nimble b/eth.nimble index 0034181..bf79a24 100644 --- a/eth.nimble +++ b/eth.nimble @@ -77,7 +77,7 @@ task test_utp, "Run utp tests": runTest("tests/utp/all_utp_tests") task test_common, "Run common tests": - runTest("tests/common/test_eth_types") + runTest("tests/common/all_tests") task test, "Run all tests": for filename in [ diff --git a/eth/common/eth_types.nim b/eth/common/eth_types.nim index 96122bd..1e2226a 100644 --- a/eth/common/eth_types.nim +++ b/eth/common/eth_types.nim @@ -62,10 +62,29 @@ type AccessList* = seq[AccessPair] + VersionedHash* = Hash256 + VersionedHashes* = seq[VersionedHash] + KzgCommitment* = array[48, byte] + KzgProof* = array[48, byte] + + # 32 -> UInt256 + # 4096 -> FIELD_ELEMENTS_PER_BLOB + NetworkBlob* = array[32*4096, byte] + TxType* = enum - TxLegacy - TxEip2930 - TxEip1559 + TxLegacy # 0 + TxEip2930 # 1 + TxEip1559 # 2 + TxEip4844 # 3 + + # instead of wrap Transaction with + # NetworkPayload, we embed it to Transaction + # the rest of magic happened in RLP + # encoding decoding + NetworkPayload* = ref object + blobs* : seq[NetworkBlob] + commitments* : seq[KzgCommitment] + proofs* : seq[KzgProof] Transaction* = object txType* : TxType # EIP-2718 @@ -79,6 +98,9 @@ type value* : UInt256 payload* : Blob accessList* : AccessList # EIP-2930 + maxFeePerDataGas*: GasInt # EIP-4844 + versionedHashes*: VersionedHashes # EIP-4844 + networkPayload*: NetworkPayload # EIP-4844 V* : int64 R*, S* : UInt256 @@ -138,6 +160,7 @@ type # LegacyReceipt = TxLegacy # Eip2930Receipt = TxEip2930 # Eip1559Receipt = TxEip1559 + # Eip4844Receipt = TxEip4844 Receipt* = object receiptType* : ReceiptType @@ -183,6 +206,7 @@ const LegacyReceipt* = TxLegacy Eip2930Receipt* = TxEip2930 Eip1559Receipt* = TxEip1559 + Eip4844Receipt* = TxEip4844 # TODO clean these up EMPTY_ROOT_HASH* = "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421".toDigest @@ -271,8 +295,12 @@ func destination*(tx: Transaction): EthAddress = if tx.to.isSome: return tx.to.get -func init*( - T: type BlockHashOrNumber, str: string): T {.raises: [ValueError].} = +func removeNetworkPayload*(tx: Transaction): Transaction = + result = tx + result.networkPayload = nil + +func init*(T: type BlockHashOrNumber, str: string): T + {.raises: [ValueError].} = if str.startsWith "0x": if str.len != sizeof(default(T).hash.data) * 2 + 2: raise newException(ValueError, "Block hash has incorrect length") diff --git a/eth/common/eth_types_rlp.nim b/eth/common/eth_types_rlp.nim index 815ee1c..df88612 100644 --- a/eth/common/eth_types_rlp.nim +++ b/eth/common/eth_types_rlp.nim @@ -83,7 +83,7 @@ proc appendTxLegacy(w: var RlpWriter, tx: Transaction) = w.append(tx.S) proc appendTxEip2930(w: var RlpWriter, tx: Transaction) = - w.append(1) + w.append(TxEip2930) w.startList(11) w.append(tx.chainId.uint64) w.append(tx.nonce) @@ -98,7 +98,7 @@ proc appendTxEip2930(w: var RlpWriter, tx: Transaction) = w.append(tx.S) proc appendTxEip1559(w: var RlpWriter, tx: Transaction) = - w.append(2) + w.append(TxEip1559) w.startList(12) w.append(tx.chainId.uint64) w.append(tx.nonce) @@ -113,6 +113,42 @@ proc appendTxEip1559(w: var RlpWriter, tx: Transaction) = w.append(tx.R) w.append(tx.S) +proc appendTxEip4844Signed(w: var RlpWriter, tx: Transaction) = + # exclude tx type + w.startList(14) + w.append(tx.chainId.uint64) + w.append(tx.nonce) + w.append(tx.maxPriorityFee) + w.append(tx.maxFee) + w.append(tx.gasLimit) + w.append(tx.to) + w.append(tx.value) + w.append(tx.payload) + w.append(tx.accessList) + w.append(tx.maxFeePerDataGas) + w.append(tx.versionedHashes) + w.append(tx.V) + w.append(tx.R) + w.append(tx.S) + +proc appendTxEip4844Network(w: var RlpWriter, tx: Transaction) = + # exclude tx type + # spec: rlp([tx_payload, blobs, commitments, proofs]) + w.startList(4) + w.appendTxEip4844Signed(tx) + w.append(tx.networkPayload.blobs) + w.append(tx.networkPayload.commitments) + w.append(tx.networkPayload.proofs) + +proc appendTxEip4844(w: var RlpWriter, tx: Transaction) = + # append the tx type first + w.append(TxEip4844) + + if tx.networkPayload.isNil: + w.appendTxEip4844Signed(tx) + else: + w.appendTxEip4844Network(tx) + proc append*(w: var RlpWriter, tx: Transaction) = case tx.txType of TxLegacy: @@ -121,6 +157,8 @@ proc append*(w: var RlpWriter, tx: Transaction) = w.appendTxEip2930(tx) of TxEip1559: w.appendTxEip1559(tx) + of TxEip4844: + w.appendTxEip4844(tx) template read[T](rlp: var Rlp, val: var T)= val = rlp.read(type val) @@ -175,6 +213,44 @@ proc readTxEip1559(rlp: var Rlp, tx: var Transaction)= rlp.read(tx.R) rlp.read(tx.S) +proc readTxEip4844Signed(rlp: var Rlp, tx: var Transaction) = + rlp.tryEnterList() + tx.chainId = rlp.read(uint64).ChainId + rlp.read(tx.nonce) + rlp.read(tx.maxPriorityFee) + rlp.read(tx.maxFee) + rlp.read(tx.gasLimit) + rlp.read(tx.to) + rlp.read(tx.value) + rlp.read(tx.payload) + rlp.read(tx.accessList) + rlp.read(tx.maxFeePerDataGas) + rlp.read(tx.versionedHashes) + rlp.read(tx.V) + rlp.read(tx.R) + rlp.read(tx.S) + +proc readTxEip4844Network(rlp: var Rlp, tx: var Transaction) = + # spec: rlp([tx_payload, blobs, commitments, proofs]) + rlp.tryEnterList() + rlp.readTxEip4844Signed(tx) + var np = NetworkPayload() + rlp.read(np.blobs) + rlp.read(np.commitments) + rlp.read(np.proofs) + tx.networkPayload = np + +proc readTxEip4844(rlp: var Rlp, tx: var Transaction) = + tx.txType = TxEip4844 + let listLen = rlp.listLen + if listLen == 4: + rlp.readTxEip4844Network(tx) + elif listLen == 14: + rlp.readTxEip4844Signed(tx) + else: + raise newException(MalformedRlpError, + "Invalid EIP-4844 transaction: listLen should be in 4 or 14, got: " & $listLen) + proc readTxTyped(rlp: var Rlp, tx: var Transaction) {.inline.} = # EIP-2718: We MUST decode the first byte as a byte, not `rlp.read(int)`. # If decoded with `rlp.read(int)`, bad transaction data (from the network) @@ -204,12 +280,14 @@ proc readTxTyped(rlp: var Rlp, tx: var Transaction) {.inline.} = of TxEip1559: rlp.readTxEip1559(tx) return + of TxEip4844: + rlp.readTxEip4844(tx) + return else: discard raise newException(UnsupportedRlpError, - "TypedTransaction type must be 1 or 2 in this version, got " & $txType) - + "TypedTransaction type must be 1, 2, or 3 in this version, got " & $txType) proc read*(rlp: var Rlp, T: type Transaction): T = # Individual transactions are encoded and stored as either `RLP([fields..])` @@ -258,7 +336,7 @@ proc append*(rlpWriter: var RlpWriter, rlpWriter.append(rlp.encode(tx)) proc append*(w: var RlpWriter, rec: Receipt) = - if rec.receiptType in {Eip2930Receipt, Eip1559Receipt}: + if rec.receiptType in {Eip2930Receipt, Eip1559Receipt, Eip4844Receipt}: w.append(rec.receiptType.int) w.startList(4) @@ -276,10 +354,12 @@ proc read*(rlp: var Rlp, T: type Receipt): T = result.receiptType = LegacyReceipt else: # EIP 2718 - let recType = rlp.read(int) - if recType notin {1, 2}: + let recType = rlp.getByteValue + rlp.position += 1 + + if recType notin {1, 2, 3}: raise newException(UnsupportedRlpError, - "TxType expect 1 or 2 got " & $recType) + "TxType expect 1, 2, or 3 got " & $recType) result.receiptType = ReceiptType(recType) rlp.tryEnterList() diff --git a/eth/common/transaction.nim b/eth/common/transaction.nim index cf696a3..528f76a 100644 --- a/eth/common/transaction.nim +++ b/eth/common/transaction.nim @@ -1,5 +1,5 @@ import - ../common/eth_types_rlp + ./eth_types_rlp export eth_types_rlp @@ -58,7 +58,7 @@ func rlpEncodeEip155(tx: Transaction): auto = func rlpEncodeEip2930(tx: Transaction): auto = var w = initRlpWriter() - w.append(1) + w.append(TxEip2930) w.startList(8) w.append(tx.chainId.uint64) w.append(tx.nonce) @@ -72,7 +72,7 @@ func rlpEncodeEip2930(tx: Transaction): auto = func rlpEncodeEip1559(tx: Transaction): auto = var w = initRlpWriter() - w.append(2) + w.append(TxEip1559) w.startList(9) w.append(tx.chainId.uint64) w.append(tx.nonce) @@ -85,6 +85,23 @@ func rlpEncodeEip1559(tx: Transaction): auto = w.append(tx.accessList) w.finish() +func rlpEncodeEip4844(tx: Transaction): auto = + var w = initRlpWriter() + w.append(TxEip4844) + w.startList(11) + w.append(tx.chainId.uint64) + w.append(tx.nonce) + w.append(tx.maxPriorityFee) + w.append(tx.maxFee) + w.append(tx.gasLimit) + w.append(tx.to) + w.append(tx.value) + w.append(tx.payload) + w.append(tx.accessList) + w.append(tx.maxFeePerDataGas) + w.append(tx.versionedHashes) + w.finish() + func rlpEncode*(tx: Transaction): auto = case tx.txType of TxLegacy: @@ -96,6 +113,8 @@ func rlpEncode*(tx: Transaction): auto = tx.rlpEncodeEip2930 of TxEip1559: tx.rlpEncodeEip1559 + of TxEip4844: + tx.rlpEncodeEip4844 func txHashNoSignature*(tx: Transaction): Hash256 = # Hash transaction without signature diff --git a/eth/common/utils.nim b/eth/common/utils.nim index b6acc9f..603b00c 100644 --- a/eth/common/utils.nim +++ b/eth/common/utils.nim @@ -7,12 +7,12 @@ # those terms. import - std/[hashes, os], + std/[hashes], nimcrypto/hash, stew/byteutils, metrics, ./eth_types when defined(posix): - import std/posix + import std/[posix, os] export metrics diff --git a/eth/p2p/private/p2p_types.nim b/eth/p2p/private/p2p_types.nim index 005bf77..6956597 100644 --- a/eth/p2p/private/p2p_types.nim +++ b/eth/p2p/private/p2p_types.nim @@ -153,18 +153,28 @@ type # Private types: MessageHandlerDecorator* = proc(msgId: int, n: NimNode): NimNode + ThunkProc* = proc(x: Peer, msgId: int, data: Rlp): Future[void] {.gcsafe, raises: [RlpError].} + MessageContentPrinter* = proc(msg: pointer): string {.gcsafe, raises: [].} + RequestResolver* = proc(msg: pointer, future: FutureBase) {.gcsafe, raises: [].} + NextMsgResolver* = proc(msgData: Rlp, future: FutureBase) {.gcsafe, raises: [RlpError].} - PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe, raises: [].} + + PeerStateInitializer* = proc(peer: Peer): RootRef + {.gcsafe, raises: [].} + NetworkStateInitializer* = proc(network: EthereumNode): RootRef {.gcsafe, raises: [].} - HandshakeStep* = proc(peer: Peer): Future[void] {.gcsafe, raises: [].} + + HandshakeStep* = proc(peer: Peer): Future[void] + {.gcsafe, raises: [].} + DisconnectionHandler* = proc(peer: Peer, reason: DisconnectionReason): Future[void] {.gcsafe, raises: [].} diff --git a/eth/rlp.nim b/eth/rlp.nim index 8ad0213..8731ae3 100644 --- a/eth/rlp.nim +++ b/eth/rlp.nim @@ -53,6 +53,9 @@ proc currentElemEnd*(self: Rlp): int {.gcsafe.} template rawData*(self: Rlp): openArray[byte] = self.bytes.toOpenArray(self.position, self.currentElemEnd - 1) +template remainingBytes*(self: Rlp): openArray[byte] = + self.bytes.toOpenArray(self.position, self.bytes.len - 1) + proc isBlob*(self: Rlp): bool = self.hasData() and self.bytes[self.position] < LIST_START_MARKER diff --git a/tests/common/all_tests.nim b/tests/common/all_tests.nim new file mode 100644 index 0000000..b3cfe67 --- /dev/null +++ b/tests/common/all_tests.nim @@ -0,0 +1,14 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +import + test_eth_types, + test_eip4844 + \ No newline at end of file diff --git a/tests/common/test_eip4844.nim b/tests/common/test_eip4844.nim new file mode 100644 index 0000000..e974772 --- /dev/null +++ b/tests/common/test_eip4844.nim @@ -0,0 +1,236 @@ +# Nimbus +# Copyright (c) 2023 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or +# http://www.apache.org/licenses/LICENSE-2.0) +# * MIT license ([LICENSE-MIT](LICENSE-MIT) or +# http://opensource.org/licenses/MIT) +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. +{.used.} + +import + stew/byteutils, + unittest2, + ../../eth/common, + ../../eth/rlp, + ../../eth/common/transaction + + +const + recipient = hexToByteArray[20]("095e7baea6a6c7c4c2dfeb977efac326af552d87") + zeroG1 = hexToByteArray[48]("0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + source = hexToByteArray[20]("0x0000000000000000000000000000000000000001") + storageKey= default(StorageKey) + accesses = @[AccessPair(address: source, storageKeys: @[storageKey])] + blob = default(NetworkBlob) + abcdef = hexToSeqByte("abcdef") + +proc tx0(i: int): Transaction = + Transaction( + txType: TxLegacy, + nonce: i.AccountNonce, + to: recipient.some, + gasLimit: 1.GasInt, + gasPrice: 2.GasInt, + payload: abcdef) + +proc tx1(i: int): Transaction = + Transaction( + # Legacy tx contract creation. + txType: TxLegacy, + nonce: i.AccountNonce, + gasLimit: 1.GasInt, + gasPrice: 2.GasInt, + payload: abcdef) + +proc tx2(i: int): Transaction = + Transaction( + # Tx with non-zero access list. + txType: TxEip2930, + chainId: 1.ChainId, + nonce: i.AccountNonce, + to: recipient.some, + gasLimit: 123457.GasInt, + gasPrice: 10.GasInt, + accessList: accesses, + payload: abcdef) + +proc tx3(i: int): Transaction = + Transaction( + # Tx with empty access list. + txType: TxEip2930, + chainId: 1.ChainId, + nonce: i.AccountNonce, + to: recipient.some, + gasLimit: 123457.GasInt, + gasPrice: 10.GasInt, + payload: abcdef) + +proc tx4(i: int): Transaction = + Transaction( + # Contract creation with access list. + txType: TxEip2930, + chainId: 1.ChainId, + nonce: i.AccountNonce, + gasLimit: 123457.GasInt, + gasPrice: 10.GasInt, + accessList: accesses) + +proc tx5(i: int): Transaction = + Transaction( + txType: TxEip1559, + chainId: 1.ChainId, + nonce: i.AccountNonce, + gasLimit: 123457.GasInt, + maxPriorityFee: 42.GasInt, + maxFee: 10.GasInt, + accessList: accesses) + +proc tx6(i: int): Transaction = + const + digest = "010657f37554c781402a22917dee2f75def7ab966d7b770905398eba3c444014".toDigest + + Transaction( + txType: TxEip4844, + chainId: 1.ChainId, + nonce: i.AccountNonce, + gasLimit: 123457.GasInt, + maxPriorityFee: 42.GasInt, + maxFee: 10.GasInt, + accessList: accesses, + versionedHashes: @[digest], + networkPayload: NetworkPayload( + blobs: @[blob], + commitments: @[zeroG1], + proofs: @[zeroG1], + ) + ) + +proc tx7(i: int): Transaction = + const + digest = "01624652859a6e98ffc1608e2af0147ca4e86e1ce27672d8d3f3c9d4ffd6ef7e".toDigest + + Transaction( + txType: TxEip4844, + chainID: 1.ChainId, + nonce: i.AccountNonce, + gasLimit: 123457.GasInt, + maxPriorityFee: 42.GasInt, + maxFee: 10.GasInt, + accessList: accesses, + versionedHashes: @[digest], + maxFeePerDataGas: 10000000.GasInt, + ) + +proc tx8(i: int): Transaction = + const + digest = "01624652859a6e98ffc1608e2af0147ca4e86e1ce27672d8d3f3c9d4ffd6ef7e".toDigest + + Transaction( + txType: TxEip4844, + chainID: 1.ChainId, + nonce: i.AccountNonce, + to: some(recipient), + gasLimit: 123457.GasInt, + maxPriorityFee: 42.GasInt, + maxFee: 10.GasInt, + accessList: accesses, + versionedHashes: @[digest], + maxFeePerDataGas: 10000000.GasInt, + ) + +template roundTrip(txFunc: untyped, i: int) = + let tx = txFunc(i) + let bytes = rlp.encode(tx) + let tx2 = rlp.decode(bytes, Transaction) + let bytes2 = rlp.encode(tx2) + check bytes == bytes2 + +suite "Transaction RLP Encoding": + test "Legacy Tx Call": + roundTrip(tx0, 1) + + test "Legacy tx contract creation": + roundTrip(tx1, 2) + + test "Tx with non-zero access list": + roundTrip(tx2, 3) + + test "Tx with empty access list": + roundTrip(tx3, 4) + + test "Contract creation with access list": + roundTrip(tx4, 5) + + test "Dynamic Fee Tx": + roundTrip(tx5, 6) + + test "NetworkBlob Tx": + roundTrip(tx6, 7) + + test "Minimal Blob Tx": + roundTrip(tx7, 8) + + test "Minimal Blob Tx contract creation": + roundTrip(tx8, 9) + + test "Network payload survive encode decode": + let tx = tx6(10) + let bytes = rlp.encode(tx) + let zz = rlp.decode(bytes, Transaction) + check not zz.networkPayload.isNil + check zz.networkPayload.proofs == tx.networkPayload.proofs + check zz.networkPayload.blobs == tx.networkPayload.blobs + check zz.networkPayload.commitments == tx.networkPayload.commitments + + test "No Network payload still no network payload": + let tx = tx7(11) + let bytes = rlp.encode(tx) + let zz = rlp.decode(bytes, Transaction) + check zz.networkPayload.isNil + + test "Minimal Blob tx recipient survive encode decode": + let tx = tx8(12) + let bytes = rlp.encode(tx) + let zz = rlp.decode(bytes, Transaction) + check zz.to.isSome + + test "Tx List 0,1,2,3,4,5,6,7,8": + let txs = @[tx0(3), tx1(3), tx2(3), tx3(3), tx4(3), + tx5(3), tx6(3), tx7(3), tx8(3)] + + let bytes = rlp.encode(txs) + let zz = rlp.decode(bytes, seq[Transaction]) + let bytes2 = rlp.encode(zz) + check bytes2 == bytes + + test "Tx List 8,7,6,5,4,3,2,1,0": + let txs = @[tx8(3), tx7(3) , tx6(3), tx5(3), tx4(3), + tx3(3), tx2(3), tx1(3), tx0(3)] + + let bytes = rlp.encode(txs) + let zz = rlp.decode(bytes, seq[Transaction]) + let bytes2 = rlp.encode(zz) + check bytes2 == bytes + + test "Tx List 0,5,8,7,6,4,3,2,1": + let txs = @[tx0(3), tx5(3), tx8(3), tx7(3), tx6(3), + tx4(3), tx3(3), tx2(3), tx1(3)] + + let bytes = rlp.encode(txs) + let zz = rlp.decode(bytes, seq[Transaction]) + let bytes2 = rlp.encode(zz) + check bytes2 == bytes + + test "Receipts": + let rec = Receipt( + receiptType: Eip4844Receipt, + isHash: false, + status: false, + cumulativeGasUsed: 100.GasInt) + + let bytes = rlp.encode(rec) + let zz = rlp.decode(bytes, Receipt) + let bytes2 = rlp.encode(zz) + check bytes2 == bytes diff --git a/tests/common/test_eth_types.nim b/tests/common/test_eth_types.nim index a4fb66c..6e04628 100644 --- a/tests/common/test_eth_types.nim +++ b/tests/common/test_eth_types.nim @@ -1,3 +1,5 @@ +{.used.} + import unittest2, nimcrypto/hash,