Fix blobs (#2762)

* add blobGasUsed calculation

* enable old tests

* add blob tests in kurtosis

* add blobGasUsed validation

* revert

* introduce blobs in test

* fix: kzg setup

* prevent regression

kurtosis-network-params.yml

@@ -20,11 +20,12 @@ additional_services:
   - tx_spammer
   - assertoor
   - beacon_metrics_gazer
+  - blob_spammer
 mev_type: null
 assertoor_params:
   image: "ethpandaops/assertoor:latest"
   run_stability_check: false
   run_block_proposal_check: true
   run_transaction_test: true
-  run_blob_transaction_test: false
+  run_blob_transaction_test: true
   run_opcodes_transaction_test: true

nimbus/core/tx_pool/tx_packer.nim

@@ -203,6 +203,11 @@ proc vmExecGrabItem(pst: var TxPacker; item: TxItemRef): GrabResult
   if pst.numBlobPerBlock + item.tx.versionedHashes.len > MAX_BLOBS_PER_BLOCK:
     return ContinueWithNextAccount
   pst.numBlobPerBlock += item.tx.versionedHashes.len
+  
+  let blobGasUsed = item.tx.getTotalBlobGas
+  if vmState.blobGasUsed + blobGasUsed > MAX_BLOB_GAS_PER_BLOCK:
+    return ContinueWithNextAccount
+  vmState.blobGasUsed += blobGasUsed
 
   # Verify we have enough gas in gasPool
   if vmState.gasPool < item.tx.gasLimit:

tests/test_txpool2.nim

@@ -9,11 +9,14 @@
 # according to those terms.
 
 import
-  std/tables,
+  std/[tables, math],
   eth/common/keys,
-  results, unittest2,
+  results,
+  unittest2,
+  ../hive_integration/nodocker/engine/tx_sender,
   ../nimbus/db/ledger,
   ../nimbus/core/chain,
+  ../nimbus/core/eip4844,
   ../nimbus/[config, transaction, constants],
   ../nimbus/core/tx_pool,
   ../nimbus/core/tx_pool/tx_desc,
@@ -29,15 +32,14 @@ const
   repoDir = [".", "customgenesis"]
   genesisFile = "merge.json"
 
-type
+type TestEnv = object
-  TestEnv = object
+  nonce: uint64
-    nonce   : uint64
+  chainId: ChainId
-    chainId : ChainId
+  vaultKey: PrivateKey
-    vaultKey: PrivateKey
+  conf: NimbusConf
-    conf    : NimbusConf
+  com: CommonRef
-    com     : CommonRef
+  chain: ForkedChainRef
-    chain   : ForkedChainRef
+  xp: TxPoolRef
-    xp      : TxPoolRef
 
 const
   # signerKeyHex = "9c647b8b7c4e7c3490668fb6c11473619db80c93704c70893d3813af4090c39c"
@@ -45,9 +47,9 @@ const
   recipient = address"0000000000000000000000000000000000000318"
   feeRecipient = address"0000000000000000000000000000000000000212"
   contractCode = evmByteCode:
-    PrevRandao    # VAL
+    PrevRandao # VAL
-    Push1 "0x11"  # KEY
+    Push1 "0x11" # KEY
-    Sstore        # OP
+    Sstore # OP
     Stop
 
 proc privKey(keyHex: string): PrivateKey =
@@ -59,42 +61,57 @@ proc privKey(keyHex: string): PrivateKey =
   kRes.get()
 
 func makeTx(
-    t: var TestEnv, recipient: Address, amount: UInt256,
+    t: var TestEnv, recipient: Address, amount: UInt256, payload: openArray[byte] = []
-    payload: openArray[byte] = []): Transaction =
+): Transaction =
   const
     gasLimit = 75000.GasInt
     gasPrice = 30.gwei
 
   let tx = Transaction(
-    txType  : TxLegacy,
+    txType: TxLegacy,
-    chainId : t.chainId,
+    chainId: t.chainId,
-    nonce   : AccountNonce(t.nonce),
+    nonce: AccountNonce(t.nonce),
     gasPrice: gasPrice,
     gasLimit: gasLimit,
-    to      : Opt.some(recipient),
+    to: Opt.some(recipient),
-    value   : amount,
+    value: amount,
-    payload : @payload
+    payload: @payload,
   )
 
   inc t.nonce
   signTransaction(tx, t.vaultKey, eip155 = true)
 
-func signTxWithNonce(
+proc createPooledTransactionWithBlob(
-    t: TestEnv, tx: Transaction, nonce: AccountNonce): Transaction =
+    t: var TestEnv, recipient: Address, amount: UInt256
+): PooledTransaction =
+  # Create the transaction
+  let
+    tc = BlobTx(
+      recipient: Opt.some(recipient),
+      gasLimit: 100000.GasInt,
+      gasTip: GasInt(10 ^ 9),
+      gasFee: GasInt(10 ^ 9),
+      blobGasFee: u256(1),
+      blobCount: 1,
+      blobID: 1,
+    )
+    params = MakeTxParams(chainId: t.chainId, key: t.vaultKey, nonce: t.nonce)
+
+  inc t.nonce
+  params.makeTx(tc)
+
+func signTxWithNonce(t: TestEnv, tx: Transaction, nonce: AccountNonce): Transaction =
   var tx = tx
   tx.nonce = nonce
   signTransaction(tx, t.vaultKey, eip155 = true)
 
 proc initEnv(envFork: HardFork): TestEnv =
-  var
+  var conf = makeConfig(
-    conf = makeConfig(@[
+    @["--custom-network:" & genesisFile.findFilePath(baseDir, repoDir).value]
-      "--custom-network:" & genesisFile.findFilePath(baseDir,repoDir).value
-    ])
-
-  conf.networkParams.genesis.alloc[recipient] = GenesisAccount(
-    code: contractCode
   )
 
+  conf.networkParams.genesis.alloc[recipient] = GenesisAccount(code: contractCode)
+
   if envFork >= MergeFork:
     conf.networkParams.config.mergeForkBlock = Opt.some(0'u64)
     conf.networkParams.config.terminalTotalDifficulty = Opt.some(100.u256)
@@ -106,11 +123,8 @@ proc initEnv(envFork: HardFork): TestEnv =
     conf.networkParams.config.cancunTime = Opt.some(0.EthTime)
 
   let
-    com = CommonRef.new(
+    com =
-      newCoreDbRef DefaultDbMemory,
+      CommonRef.new(newCoreDbRef DefaultDbMemory, conf.networkId, conf.networkParams)
-      conf.networkId,
-      conf.networkParams
-    )
     chain = newForkedChain(com, com.genesisHeader)
 
   result = TestEnv(
@@ -120,7 +134,7 @@ proc initEnv(envFork: HardFork): TestEnv =
     xp: TxPoolRef.new(com),
     vaultKey: privKey(vaultKeyHex),
     chainId: conf.networkParams.config.chainId,
-    nonce: 0'u64
+    nonce: 0'u64,
   )
 
 const
@@ -129,8 +143,7 @@ const
   prevRandao = Bytes32 EMPTY_UNCLE_HASH # it can be any valid hash
 
 proc runTxPoolPosTest() =
-  var
+  var env = initEnv(MergeFork)
-    env = initEnv(MergeFork)
 
   var
     tx = env.makeTx(recipient, amount)
@@ -159,10 +172,7 @@ proc runTxPoolPosTest() =
         return
 
       blk = r.get.blk
-      body = BlockBody(
+      body = BlockBody(transactions: blk.txs, uncles: blk.uncles)
-        transactions: blk.txs,
-        uncles: blk.uncles
-      )
       check blk.txs.len == 1
 
     test "PoS persistBlocks":
@@ -182,12 +192,11 @@ proc runTxPoolPosTest() =
       check not bal.isZero
 
 proc runTxPoolBlobhashTest() =
-  var
+  var env = initEnv(Cancun)
-    env = initEnv(Cancun)
 
   var
-    tx1 = env.makeTx(recipient, amount)
+    tx1 = env.createPooledTransactionWithBlob(recipient, amount)
-    tx2 = env.makeTx(recipient, amount)
+    tx2 = env.createPooledTransactionWithBlob(recipient, amount)
     xp = env.xp
     com = env.com
     chain = env.chain
@@ -196,8 +205,8 @@ proc runTxPoolBlobhashTest() =
 
   suite "Test TxPool with blobhash block":
     test "TxPool jobCommit":
-      xp.add(PooledTransaction(tx: tx1))
+      xp.add(tx1)
-      xp.add(PooledTransaction(tx: tx2))
+      xp.add(tx2)
       check xp.nItems.total == 2
 
     test "TxPool ethBlock":
@@ -216,17 +225,20 @@ proc runTxPoolBlobhashTest() =
       body = BlockBody(
         transactions: blk.txs,
         uncles: blk.uncles,
-        withdrawals: Opt.some(newSeq[Withdrawal]())
+        withdrawals: Opt.some(newSeq[Withdrawal]()),
       )
       check blk.txs.len == 2
 
       let
         gasUsed1 = xp.vmState.receipts[0].cumulativeGasUsed
         gasUsed2 = xp.vmState.receipts[1].cumulativeGasUsed - gasUsed1
-        blockValue = gasUsed1.u256 * tx1.effectiveGasTip(blk.header.baseFeePerGas).u256 +
+        totalBlobGasUsed = tx1.tx.getTotalBlobGas + tx2.tx.getTotalBlobGas
-          gasUsed2.u256 * tx2.effectiveGasTip(blk.header.baseFeePerGas).u256
+        blockValue =
+          gasUsed1.u256 * tx1.tx.effectiveGasTip(blk.header.baseFeePerGas).u256 +
+          gasUsed2.u256 * tx2.tx.effectiveGasTip(blk.header.baseFeePerGas).u256
 
       check blockValue == bundle.blockValue
+      check totalBlobGasUsed == blk.header.blobGasUsed.get()
 
     test "Blobhash persistBlocks":
       let rr = chain.importBlock(EthBlock.init(blk.header, body))
@@ -247,7 +259,7 @@ proc runTxPoolBlobhashTest() =
     test "add tx with nonce too low":
       let
         tx3 = env.makeTx(recipient, amount)
-        tx4 = env.signTxWithNonce(tx3, AccountNonce(env.nonce-2))
+        tx4 = env.signTxWithNonce(tx3, AccountNonce(env.nonce - 2))
         xp = env.xp
 
       check xp.smartHead(blk.header, chain)
@@ -275,20 +287,20 @@ proc runTxHeadDelta(noisy = true) =
       # setTraceLevel()
 
       block:
-        for n in 0..<numBlocks:
+        for n in 0 ..< numBlocks:
-
+          for tn in 0 ..< txPerblock:
-          for tn in 0..<txPerblock:
             let tx = env.makeTx(recipient, amount)
             xp.add(PooledTransaction(tx: tx))
 
-          noisy.say "***", "txDB",
+          noisy.say "***",
+            "txDB",
             &" n={n}",
             # pending/staged/packed : total/disposed
             &" stats={xp.nItems.pp}"
 
           timestamp = timestamp + 1
           com.pos.prevRandao = prevRandao
-          com.pos.timestamp  = timestamp
+          com.pos.timestamp = timestamp
           com.pos.feeRecipient = feeRecipient
 
           let r = xp.assembleBlock()
@@ -298,9 +310,7 @@ proc runTxHeadDelta(noisy = true) =
             return
 
           let blk = r.get.blk
-          let body = BlockBody(
+          let body = BlockBody(transactions: blk.txs, uncles: blk.uncles)
-            transactions: blk.txs,
-            uncles: blk.uncles)
 
           # Commit to block chain
           check chain.importBlock(EthBlock.init(blk.header, body)).isOk
@@ -383,8 +393,9 @@ proc runGetBlockBodyTest() =
       check env.chain.forkChoice(currHash, currHash).isOk
 
 proc txPool2Main*() =
-  const
+  const noisy = defined(debug)
-    noisy = defined(debug)
+
+  assert loadKzgTrustedSetup().isOk, "Failed to load KZG trusted setup"
 
   setErrorLevel() # mute logger