implement EIP-4844: Shard Blobs Transactions

eth.nimble

@@ -77,7 +77,7 @@ task test_utp, "Run utp tests":
   runTest("tests/utp/all_utp_tests")
 
 task test_common, "Run common tests":
-  runTest("tests/common/test_eth_types")
+  runTest("tests/common/all_tests")
 
 task test, "Run all tests":
   for filename in [

eth/common/eth_types.nim

@@ -62,10 +62,29 @@ type
 
   AccessList* = seq[AccessPair]
 
+  VersionedHash* = Hash256
+  VersionedHashes* = seq[VersionedHash]
+  KzgCommitment* = array[48, byte]
+  KzgProof* = array[48, byte]
+
+  # 32 -> UInt256
+  # 4096 -> FIELD_ELEMENTS_PER_BLOB
+  NetworkBlob* = array[32*4096, byte]
+
   TxType* = enum
-    TxLegacy
+    TxLegacy    # 0
-    TxEip2930
+    TxEip2930   # 1
-    TxEip1559
+    TxEip1559   # 2
+    TxEip4844   # 3
+
+  # instead of wrap Transaction with
+  # NetworkPayload, we embed it to Transaction
+  # the rest of magic happened in RLP
+  # encoding decoding
+  NetworkPayload* = ref object
+    blobs*       : seq[NetworkBlob]
+    commitments* : seq[KzgCommitment]
+    proofs*      : seq[KzgProof]
 
   Transaction* = object
     txType*        : TxType               # EIP-2718
@@ -79,6 +98,9 @@ type
     value*         : UInt256
     payload*       : Blob
     accessList*    : AccessList           # EIP-2930
+    maxFeePerDataGas*: GasInt             # EIP-4844
+    versionedHashes*: VersionedHashes     # EIP-4844
+    networkPayload*: NetworkPayload       # EIP-4844
     V*             : int64
     R*, S*         : UInt256
 
@@ -138,6 +160,7 @@ type
     # LegacyReceipt  = TxLegacy
     # Eip2930Receipt = TxEip2930
     # Eip1559Receipt = TxEip1559
+    # Eip4844Receipt = TxEip4844
 
   Receipt* = object
     receiptType*      : ReceiptType
@@ -183,6 +206,7 @@ const
   LegacyReceipt*  = TxLegacy
   Eip2930Receipt* = TxEip2930
   Eip1559Receipt* = TxEip1559
+  Eip4844Receipt* = TxEip4844
 
   # TODO clean these up
   EMPTY_ROOT_HASH* = "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421".toDigest
@@ -271,8 +295,12 @@ func destination*(tx: Transaction): EthAddress =
   if tx.to.isSome:
     return tx.to.get
 
-func init*(
+func removeNetworkPayload*(tx: Transaction): Transaction =
-    T: type BlockHashOrNumber, str: string): T {.raises: [ValueError].} =
+  result = tx
+  result.networkPayload = nil
+
+func init*(T: type BlockHashOrNumber, str: string): T
+          {.raises: [ValueError].} =
   if str.startsWith "0x":
     if str.len != sizeof(default(T).hash.data) * 2 + 2:
       raise newException(ValueError, "Block hash has incorrect length")

eth/common/eth_types_rlp.nim

@@ -83,7 +83,7 @@ proc appendTxLegacy(w: var RlpWriter, tx: Transaction) =
   w.append(tx.S)
 
 proc appendTxEip2930(w: var RlpWriter, tx: Transaction) =
-  w.append(1)
+  w.append(TxEip2930)
   w.startList(11)
   w.append(tx.chainId.uint64)
   w.append(tx.nonce)
@@ -98,7 +98,7 @@ proc appendTxEip2930(w: var RlpWriter, tx: Transaction) =
   w.append(tx.S)
 
 proc appendTxEip1559(w: var RlpWriter, tx: Transaction) =
-  w.append(2)
+  w.append(TxEip1559)
   w.startList(12)
   w.append(tx.chainId.uint64)
   w.append(tx.nonce)
@@ -113,6 +113,42 @@ proc appendTxEip1559(w: var RlpWriter, tx: Transaction) =
   w.append(tx.R)
   w.append(tx.S)
 
+proc appendTxEip4844Signed(w: var RlpWriter, tx: Transaction) =
+  # exclude tx type
+  w.startList(14)
+  w.append(tx.chainId.uint64)
+  w.append(tx.nonce)
+  w.append(tx.maxPriorityFee)
+  w.append(tx.maxFee)
+  w.append(tx.gasLimit)
+  w.append(tx.to)
+  w.append(tx.value)
+  w.append(tx.payload)
+  w.append(tx.accessList)
+  w.append(tx.maxFeePerDataGas)
+  w.append(tx.versionedHashes)
+  w.append(tx.V)
+  w.append(tx.R)
+  w.append(tx.S)
+
+proc appendTxEip4844Network(w: var RlpWriter, tx: Transaction) =
+  # exclude tx type
+  # spec: rlp([tx_payload, blobs, commitments, proofs])
+  w.startList(4)
+  w.appendTxEip4844Signed(tx)
+  w.append(tx.networkPayload.blobs)
+  w.append(tx.networkPayload.commitments)
+  w.append(tx.networkPayload.proofs)
+
+proc appendTxEip4844(w: var RlpWriter, tx: Transaction) =
+  # append the tx type first
+  w.append(TxEip4844)
+
+  if tx.networkPayload.isNil:
+    w.appendTxEip4844Signed(tx)
+  else:
+    w.appendTxEip4844Network(tx)
+
 proc append*(w: var RlpWriter, tx: Transaction) =
   case tx.txType
   of TxLegacy:
@@ -121,6 +157,8 @@ proc append*(w: var RlpWriter, tx: Transaction) =
     w.appendTxEip2930(tx)
   of TxEip1559:
     w.appendTxEip1559(tx)
+  of TxEip4844:
+    w.appendTxEip4844(tx)
 
 template read[T](rlp: var Rlp, val: var T)=
   val = rlp.read(type val)
@@ -175,6 +213,44 @@ proc readTxEip1559(rlp: var Rlp, tx: var Transaction)=
   rlp.read(tx.R)
   rlp.read(tx.S)
 
+proc readTxEip4844Signed(rlp: var Rlp, tx: var Transaction) =
+  rlp.tryEnterList()
+  tx.chainId = rlp.read(uint64).ChainId
+  rlp.read(tx.nonce)
+  rlp.read(tx.maxPriorityFee)
+  rlp.read(tx.maxFee)
+  rlp.read(tx.gasLimit)
+  rlp.read(tx.to)
+  rlp.read(tx.value)
+  rlp.read(tx.payload)
+  rlp.read(tx.accessList)
+  rlp.read(tx.maxFeePerDataGas)
+  rlp.read(tx.versionedHashes)
+  rlp.read(tx.V)
+  rlp.read(tx.R)
+  rlp.read(tx.S)
+
+proc readTxEip4844Network(rlp: var Rlp, tx: var Transaction) =
+  # spec: rlp([tx_payload, blobs, commitments, proofs])
+  rlp.tryEnterList()
+  rlp.readTxEip4844Signed(tx)
+  var np = NetworkPayload()
+  rlp.read(np.blobs)
+  rlp.read(np.commitments)
+  rlp.read(np.proofs)
+  tx.networkPayload = np
+
+proc readTxEip4844(rlp: var Rlp, tx: var Transaction) =
+  tx.txType = TxEip4844
+  let listLen = rlp.listLen
+  if listLen == 4:
+    rlp.readTxEip4844Network(tx)
+  elif listLen == 14:
+    rlp.readTxEip4844Signed(tx)
+  else:
+    raise newException(MalformedRlpError,
+      "Invalid EIP-4844 transaction: listLen should be in 4 or 14, got: " & $listLen)
+
 proc readTxTyped(rlp: var Rlp, tx: var Transaction) {.inline.} =
   # EIP-2718: We MUST decode the first byte as a byte, not `rlp.read(int)`.
   # If decoded with `rlp.read(int)`, bad transaction data (from the network)
@@ -204,12 +280,14 @@ proc readTxTyped(rlp: var Rlp, tx: var Transaction) {.inline.} =
     of TxEip1559:
       rlp.readTxEip1559(tx)
       return
+    of TxEip4844:
+      rlp.readTxEip4844(tx)
+      return
     else:
       discard
 
   raise newException(UnsupportedRlpError,
-    "TypedTransaction type must be 1 or 2 in this version, got " & $txType)
+    "TypedTransaction type must be 1, 2, or 3 in this version, got " & $txType)
-
 
 proc read*(rlp: var Rlp, T: type Transaction): T =
   # Individual transactions are encoded and stored as either `RLP([fields..])`
@@ -258,7 +336,7 @@ proc append*(rlpWriter: var RlpWriter,
       rlpWriter.append(rlp.encode(tx))
 
 proc append*(w: var RlpWriter, rec: Receipt) =
-  if rec.receiptType in {Eip2930Receipt, Eip1559Receipt}:
+  if rec.receiptType in {Eip2930Receipt, Eip1559Receipt, Eip4844Receipt}:
     w.append(rec.receiptType.int)
 
   w.startList(4)
@@ -276,10 +354,12 @@ proc read*(rlp: var Rlp, T: type Receipt): T =
     result.receiptType = LegacyReceipt
   else:
     # EIP 2718
-    let recType = rlp.read(int)
+    let recType = rlp.getByteValue
-    if recType notin {1, 2}:
+    rlp.position += 1
+
+    if recType notin {1, 2, 3}:
       raise newException(UnsupportedRlpError,
-        "TxType expect 1 or 2 got " & $recType)
+        "TxType expect 1, 2, or 3 got " & $recType)
     result.receiptType = ReceiptType(recType)
 
   rlp.tryEnterList()

eth/common/transaction.nim

@@ -1,5 +1,5 @@
 import
-  ../common/eth_types_rlp
+  ./eth_types_rlp
 
 export eth_types_rlp
 
@@ -58,7 +58,7 @@ func rlpEncodeEip155(tx: Transaction): auto =
 
 func rlpEncodeEip2930(tx: Transaction): auto =
   var w = initRlpWriter()
-  w.append(1)
+  w.append(TxEip2930)
   w.startList(8)
   w.append(tx.chainId.uint64)
   w.append(tx.nonce)
@@ -72,7 +72,7 @@ func rlpEncodeEip2930(tx: Transaction): auto =
 
 func rlpEncodeEip1559(tx: Transaction): auto =
   var w = initRlpWriter()
-  w.append(2)
+  w.append(TxEip1559)
   w.startList(9)
   w.append(tx.chainId.uint64)
   w.append(tx.nonce)
@@ -85,6 +85,23 @@ func rlpEncodeEip1559(tx: Transaction): auto =
   w.append(tx.accessList)
   w.finish()
 
+func rlpEncodeEip4844(tx: Transaction): auto =
+  var w = initRlpWriter()
+  w.append(TxEip4844)
+  w.startList(11)
+  w.append(tx.chainId.uint64)
+  w.append(tx.nonce)
+  w.append(tx.maxPriorityFee)
+  w.append(tx.maxFee)
+  w.append(tx.gasLimit)
+  w.append(tx.to)
+  w.append(tx.value)
+  w.append(tx.payload)
+  w.append(tx.accessList)
+  w.append(tx.maxFeePerDataGas)
+  w.append(tx.versionedHashes)
+  w.finish()
+
 func rlpEncode*(tx: Transaction): auto =
   case tx.txType
   of TxLegacy:
@@ -96,6 +113,8 @@ func rlpEncode*(tx: Transaction): auto =
     tx.rlpEncodeEip2930
   of TxEip1559:
     tx.rlpEncodeEip1559
+  of TxEip4844:
+    tx.rlpEncodeEip4844
 
 func txHashNoSignature*(tx: Transaction): Hash256 =
   # Hash transaction without signature

eth/common/utils.nim

@@ -7,12 +7,12 @@
 # those terms.
 
 import
-  std/[hashes, os],
+  std/[hashes],
   nimcrypto/hash, stew/byteutils, metrics,
   ./eth_types
 
 when defined(posix):
-  import std/posix
+  import std/[posix, os]
 
 export metrics
 

eth/p2p/private/p2p_types.nim

@@ -153,18 +153,28 @@ type
 
   # Private types:
   MessageHandlerDecorator* = proc(msgId: int, n: NimNode): NimNode
+
   ThunkProc* = proc(x: Peer, msgId: int, data: Rlp): Future[void]
     {.gcsafe, raises: [RlpError].}
+
   MessageContentPrinter* = proc(msg: pointer): string
     {.gcsafe, raises: [].}
+
   RequestResolver* = proc(msg: pointer, future: FutureBase)
     {.gcsafe, raises: [].}
+
   NextMsgResolver* = proc(msgData: Rlp, future: FutureBase)
     {.gcsafe, raises: [RlpError].}
-  PeerStateInitializer* = proc(peer: Peer): RootRef {.gcsafe, raises: [].}
+
+  PeerStateInitializer* = proc(peer: Peer): RootRef
+    {.gcsafe, raises: [].}
+
   NetworkStateInitializer* = proc(network: EthereumNode): RootRef
     {.gcsafe, raises: [].}
-  HandshakeStep* = proc(peer: Peer): Future[void] {.gcsafe, raises: [].}
+
+  HandshakeStep* = proc(peer: Peer): Future[void]
+    {.gcsafe, raises: [].}
+
   DisconnectionHandler* = proc(peer: Peer, reason: DisconnectionReason):
     Future[void] {.gcsafe, raises: [].}
 

eth/rlp.nim

@@ -53,6 +53,9 @@ proc currentElemEnd*(self: Rlp): int {.gcsafe.}
 template rawData*(self: Rlp): openArray[byte] =
   self.bytes.toOpenArray(self.position, self.currentElemEnd - 1)
 
+template remainingBytes*(self: Rlp): openArray[byte] =
+  self.bytes.toOpenArray(self.position, self.bytes.len - 1)
+
 proc isBlob*(self: Rlp): bool =
   self.hasData() and self.bytes[self.position] < LIST_START_MARKER
 

tests/common/all_tests.nim

@@ -0,0 +1,14 @@
+# Nimbus
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+
+import
+  test_eth_types,
+  test_eip4844
+  

tests/common/test_eip4844.nim

@@ -0,0 +1,236 @@
+# Nimbus
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
+#    http://www.apache.org/licenses/LICENSE-2.0)
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT) or
+#    http://opensource.org/licenses/MIT)
+# at your option. This file may not be copied, modified, or distributed except
+# according to those terms.
+{.used.}
+
+import
+  stew/byteutils,
+  unittest2,
+  ../../eth/common,
+  ../../eth/rlp,
+  ../../eth/common/transaction
+
+
+const
+  recipient = hexToByteArray[20]("095e7baea6a6c7c4c2dfeb977efac326af552d87")
+  zeroG1    = hexToByteArray[48]("0xc00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000")
+  source    = hexToByteArray[20]("0x0000000000000000000000000000000000000001")
+  storageKey= default(StorageKey)
+  accesses  = @[AccessPair(address: source, storageKeys: @[storageKey])]
+  blob      = default(NetworkBlob)
+  abcdef    = hexToSeqByte("abcdef")
+
+proc tx0(i: int): Transaction =
+  Transaction(
+    txType:   TxLegacy,
+    nonce:    i.AccountNonce,
+    to:       recipient.some,
+    gasLimit: 1.GasInt,
+    gasPrice: 2.GasInt,
+    payload:  abcdef)
+
+proc tx1(i: int): Transaction =
+  Transaction(
+    # Legacy tx contract creation.
+    txType:   TxLegacy,
+    nonce:    i.AccountNonce,
+    gasLimit: 1.GasInt,
+    gasPrice: 2.GasInt,
+    payload:  abcdef)
+
+proc tx2(i: int): Transaction =
+  Transaction(
+    # Tx with non-zero access list.
+    txType:     TxEip2930,
+    chainId:    1.ChainId,
+    nonce:      i.AccountNonce,
+    to:         recipient.some,
+    gasLimit:   123457.GasInt,
+    gasPrice:   10.GasInt,
+    accessList: accesses,
+    payload:    abcdef)
+
+proc tx3(i: int): Transaction =
+  Transaction(
+    # Tx with empty access list.
+    txType:   TxEip2930,
+    chainId:  1.ChainId,
+    nonce:    i.AccountNonce,
+    to:       recipient.some,
+    gasLimit: 123457.GasInt,
+    gasPrice: 10.GasInt,
+    payload:  abcdef)
+
+proc tx4(i: int): Transaction =
+  Transaction(
+    # Contract creation with access list.
+    txType:     TxEip2930,
+    chainId:    1.ChainId,
+    nonce:      i.AccountNonce,
+    gasLimit:   123457.GasInt,
+    gasPrice:   10.GasInt,
+    accessList: accesses)
+
+proc tx5(i: int): Transaction =
+  Transaction(
+    txType:     TxEip1559,
+    chainId:    1.ChainId,
+    nonce:      i.AccountNonce,
+    gasLimit:   123457.GasInt,
+    maxPriorityFee: 42.GasInt,
+    maxFee:     10.GasInt,
+    accessList: accesses)
+
+proc tx6(i: int): Transaction =
+  const
+    digest = "010657f37554c781402a22917dee2f75def7ab966d7b770905398eba3c444014".toDigest
+
+  Transaction(
+    txType:              TxEip4844,
+    chainId:             1.ChainId,
+    nonce:               i.AccountNonce,
+    gasLimit:            123457.GasInt,
+    maxPriorityFee:      42.GasInt,
+    maxFee:              10.GasInt,
+    accessList:          accesses,
+    versionedHashes:     @[digest],
+    networkPayload: NetworkPayload(        
+        blobs: @[blob],
+        commitments: @[zeroG1],
+        proofs: @[zeroG1],
+    )
+  )
+
+proc tx7(i: int): Transaction =
+  const
+    digest = "01624652859a6e98ffc1608e2af0147ca4e86e1ce27672d8d3f3c9d4ffd6ef7e".toDigest
+
+  Transaction(
+    txType:              TxEip4844,
+    chainID:             1.ChainId,
+    nonce:               i.AccountNonce,
+    gasLimit:            123457.GasInt,
+    maxPriorityFee:      42.GasInt,
+    maxFee:              10.GasInt,
+    accessList:          accesses,
+    versionedHashes:     @[digest],
+    maxFeePerDataGas:    10000000.GasInt,
+  )
+
+proc tx8(i: int): Transaction =
+  const
+    digest = "01624652859a6e98ffc1608e2af0147ca4e86e1ce27672d8d3f3c9d4ffd6ef7e".toDigest
+
+  Transaction(
+    txType:              TxEip4844,
+    chainID:             1.ChainId,
+    nonce:               i.AccountNonce,
+    to:                  some(recipient),
+    gasLimit:            123457.GasInt,
+    maxPriorityFee:      42.GasInt,
+    maxFee:              10.GasInt,
+    accessList:          accesses,
+    versionedHashes:     @[digest],
+    maxFeePerDataGas:    10000000.GasInt,
+  )
+
+template roundTrip(txFunc: untyped, i: int) =
+  let tx = txFunc(i)
+  let bytes = rlp.encode(tx)
+  let tx2 = rlp.decode(bytes, Transaction)
+  let bytes2 = rlp.encode(tx2)
+  check bytes == bytes2
+
+suite "Transaction RLP Encoding":
+  test "Legacy Tx Call":
+    roundTrip(tx0, 1)
+
+  test "Legacy tx contract creation":
+    roundTrip(tx1, 2)
+
+  test "Tx with non-zero access list":
+    roundTrip(tx2, 3)
+
+  test "Tx with empty access list":
+    roundTrip(tx3, 4)
+
+  test "Contract creation with access list":
+    roundTrip(tx4, 5)
+
+  test "Dynamic Fee Tx":
+    roundTrip(tx5, 6)
+
+  test "NetworkBlob Tx":
+    roundTrip(tx6, 7)
+
+  test "Minimal Blob Tx":
+    roundTrip(tx7, 8)
+
+  test "Minimal Blob Tx contract creation":
+    roundTrip(tx8, 9)
+
+  test "Network payload survive encode decode":
+    let tx = tx6(10)
+    let bytes = rlp.encode(tx)
+    let zz = rlp.decode(bytes, Transaction)
+    check not zz.networkPayload.isNil
+    check zz.networkPayload.proofs == tx.networkPayload.proofs
+    check zz.networkPayload.blobs == tx.networkPayload.blobs
+    check zz.networkPayload.commitments == tx.networkPayload.commitments
+
+  test "No Network payload still no network payload":
+    let tx = tx7(11)
+    let bytes = rlp.encode(tx)
+    let zz = rlp.decode(bytes, Transaction)
+    check zz.networkPayload.isNil
+
+  test "Minimal Blob tx recipient survive encode decode":
+    let tx = tx8(12)
+    let bytes = rlp.encode(tx)
+    let zz = rlp.decode(bytes, Transaction)
+    check zz.to.isSome
+
+  test "Tx List 0,1,2,3,4,5,6,7,8":
+    let txs = @[tx0(3), tx1(3), tx2(3), tx3(3), tx4(3),
+                tx5(3), tx6(3), tx7(3), tx8(3)]
+
+    let bytes = rlp.encode(txs)
+    let zz = rlp.decode(bytes, seq[Transaction])
+    let bytes2 = rlp.encode(zz)
+    check bytes2 == bytes
+
+  test "Tx List 8,7,6,5,4,3,2,1,0":
+    let txs = @[tx8(3), tx7(3) , tx6(3), tx5(3), tx4(3),
+                tx3(3), tx2(3), tx1(3), tx0(3)]
+
+    let bytes = rlp.encode(txs)
+    let zz = rlp.decode(bytes, seq[Transaction])
+    let bytes2 = rlp.encode(zz)
+    check bytes2 == bytes
+
+  test "Tx List 0,5,8,7,6,4,3,2,1":
+    let txs = @[tx0(3), tx5(3), tx8(3), tx7(3), tx6(3),
+                tx4(3), tx3(3), tx2(3), tx1(3)]
+
+    let bytes = rlp.encode(txs)
+    let zz = rlp.decode(bytes, seq[Transaction])
+    let bytes2 = rlp.encode(zz)
+    check bytes2 == bytes
+
+  test "Receipts":
+    let rec = Receipt(
+      receiptType: Eip4844Receipt,
+      isHash: false,
+      status: false,
+      cumulativeGasUsed: 100.GasInt)
+
+    let bytes = rlp.encode(rec)
+    let zz = rlp.decode(bytes, Receipt)
+    let bytes2 = rlp.encode(zz)
+    check bytes2 == bytes

tests/common/test_eth_types.nim

@@ -1,3 +1,5 @@
+{.used.}
+
 import
   unittest2,
   nimcrypto/hash,