fix persistHeaderToDb, allow POS block header to be canonical

hive_integration/nodocker/consensus/consensus_sim.nim

@@ -53,11 +53,6 @@ proc main() =
         inc stat.skipped
         continue
 
-      # TODO: fix this after #1337 fixed
-      if "Merge" in name:
-        inc stat.skipped
-        continue
-
       let cd = extractChainData(unit)
       let status = processChainData(cd)
       stat.inc(name, status)

nimbus/common/common.nim

@@ -350,7 +350,8 @@ proc initializeEmptyDb*(com: CommonRef)
     trace "Writing genesis to DB"
     doAssert(com.genesisHeader.blockNumber.isZero,
       "can't commit genesis block with number > 0")
-    discard com.db.persistHeaderToDb(com.genesisHeader, none(BlockNumber))
+    discard com.db.persistHeaderToDb(com.genesisHeader,
+      com.consensusType == ConsensusType.POS)
     doAssert(canonicalHeadHashKey().toOpenArray in trieDB)
 
 # ------------------------------------------------------------------------------

nimbus/core/chain/persist_blocks.nim

@@ -118,8 +118,7 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
           return ValidationResult.Error
 
     if NoPersistHeader notin flags:
-      let ttd = c.com.ttd
+      discard c.db.persistHeaderToDb(header, c.com.consensus == ConsensusType.POS)
-      discard c.db.persistHeaderToDb(header, ttd)
 
     if NoSaveTxs notin flags:
       discard c.db.persistTransactions(header.blockNumber, body.transactions)

nimbus/db/db_chain.nim

@@ -407,29 +407,8 @@ proc getReceipts*(db: ChainDBRef; receiptRoot: Hash256): seq[Receipt] =
     receipts.add(r)
   return receipts
 
-proc readTerminalHash*(db: ChainDBRef; h: var Hash256): bool =
+proc persistHeaderToDb*(db: ChainDBRef; header: BlockHeader, 
-  let bytes = db.db.get(terminalHashKey().toOpenArray)
+                        forceCanonical: bool): seq[BlockHeader] =
-  if bytes.len == 0:
-    return false
-  try:
-    h = rlp.decode(bytes, Hash256)
-  except RlpError:
-    return false
-
-  true
-
-proc writeTerminalHash*(db: ChainDBRef; h: Hash256) =
-  db.db.put(terminalHashKey().toOpenArray, rlp.encode(h))
-
-proc currentTerminalHeader*(db: ChainDBRef; header: var BlockHeader): bool =
-  var terminalHash: Hash256
-  if not db.readTerminalHash(terminalHash):
-    return false
-  if not db.getBlockHeader(terminalHash, header):
-    return false
-  true
-
-proc persistHeaderToDb*(db: ChainDBRef; header: BlockHeader, ttd: Option[DifficultyInt]): seq[BlockHeader] =
   let isGenesis = header.parentHash == GENESIS_PARENT_HASH
   let headerHash = header.blockHash
   if not isGenesis and not db.headerExists(header.parentHash):
@@ -449,14 +428,7 @@ proc persistHeaderToDb*(db: ChainDBRef; header: BlockHeader, ttd: Option[Difficu
   except CanonicalHeadNotFound:
     return db.setAsCanonicalChainHead(headerHash)
 
-  if ttd.isSome:
+  if score > headScore or forceCanonical:
-    let ttd = ttd.get()
-    if headScore < ttd and score >= ttd:
-      db.writeTerminalHash(headerHash)
-    if score >= ttd:
-      return db.setAsCanonicalChainHead(headerHash)
-
-  if score > headScore:
     return db.setAsCanonicalChainHead(headerHash)
 
 proc persistHeaderToDbWithoutSetHead*(db: ChainDBRef; header: BlockHeader) =

nimbus/db/storage_types.nim

@@ -12,7 +12,6 @@ type
     contractHash
     cliqueSnapshot
     transitionStatus
-    terminalHash
     safeHash
     finalizedHash
     skeletonProgress
@@ -74,10 +73,6 @@ proc transitionStatusKey*(): DbKey =
   result.data[0] = byte ord(transitionStatus)
   result.dataEndPos = uint8 1
 
-proc terminalHashKey*(): DbKey =
-  result.data[0] = byte ord(terminalHash)
-  result.dataEndPos = uint8 1
-
 proc safeHashKey*(): DbKey {.inline.} =
   result.data[0] = byte ord(safeHash)
   result.dataEndPos = uint8 1

nimbus/rpc/engine_api.nim

@@ -160,25 +160,35 @@ proc setupEngineApi*(
       blockHash = conf.terminalBlockHash
 
     let db = sealingEngine.chain.db
-    let ttd = com.ttd.get(high(common.BlockNumber))
+    let ttd = com.ttd
 
-    if conf.terminalTotalDifficulty != ttd:
+    if ttd.isNone:
-      raise newException(ValueError, "invalid ttd: EL $1 CL $2" % [$ttd, $conf.terminalTotalDifficulty])
+      raise newException(ValueError, "invalid ttd: EL (none) CL ($2)" % [$conf.terminalTotalDifficulty])
+
+    if conf.terminalTotalDifficulty != ttd.get:
+      raise newException(ValueError, "invalid ttd: EL ($1) CL ($2)" % [$ttd.get, $conf.terminalTotalDifficulty])
+
+    let terminalBlockNumber = uint64(conf.terminalBlockNumber).toBlockNumber
+    let terminalBlockHash = conf.terminalBlockHash.asEthHash
+
+    if terminalBlockHash != Hash256():
+      var headerHash: Hash256
+
+      if not db.getBlockHash(terminalBlockNumber, headerHash):
+        raise newException(ValueError, "cannot get terminal block hash, number $1" %
+          [$terminalBlockNumber])
+
+      if terminalBlockHash != headerHash:
+        raise newException(ValueError, "invalid terminal block hash, got $1 want $2" %
+          [$terminalBlockHash, $headerHash])
 
       var header: EthBlockHeader
-    let terminalBlockNumber = uint64(conf.terminalBlockNumber)
+      if not db.getBlockHeader(headerHash, header):
-    let terminalBlockHash = conf.terminalBlockHash.asEthHash
+        raise newException(ValueError, "cannot get terminal block header, hash $1" %
-    if db.currentTerminalHeader(header):
+          [$terminalBlockHash])
-      let headerHash = header.blockHash
-
-      if terminalBlockNumber != 0'u64 and terminalBlockNumber != header.blockNumber.truncate(uint64):
-        raise newException(ValueError, "invalid terminal block number, got $1 want $2" % [$terminalBlockNumber, $header.blockNumber])
-
-      if terminalBlockHash != Hash256() and terminalBlockHash != headerHash:
-        raise newException(ValueError, "invalid terminal block hash, got $1 want $2" % [$terminalBlockHash, $headerHash])
 
       return TransitionConfigurationV1(
-        terminalTotalDifficulty: ttd,
+        terminalTotalDifficulty: ttd.get,
         terminalBlockHash      : BlockHash headerHash.data,
         terminalBlockNumber    : Quantity header.blockNumber.truncate(uint64)
       )
@@ -189,7 +199,7 @@ proc setupEngineApi*(
     if terminalBlockHash != Hash256():
       raise newException(ValueError, "invalid terminal block hash, no terminal header set")
 
-    return TransitionConfigurationV1(terminalTotalDifficulty: ttd)
+    return TransitionConfigurationV1(terminalTotalDifficulty: ttd.get)
 
   # ForkchoiceUpdatedV1 has several responsibilities:
   # If the method is called with an empty head block:

tests/test_blockchain_json.nim

@@ -197,7 +197,8 @@ proc importBlock(tester: var Tester, com: CommonRef,
     if tester.vmState.generateWitness():
      blockWitness(tester.vmState, com.db)
 
-  discard com.db.persistHeaderToDb(tb.header, none(DifficultyInt))
+  discard com.db.persistHeaderToDb(tb.header,
+    com.consensus == ConsensusType.POS)
 
 proc applyFixtureBlockToChain(tester: var Tester, tb: var TestBlock,
                               com: CommonRef, checkSeal, validation: bool) =
@@ -222,7 +223,8 @@ proc collectDebugData(tester: var Tester) =
   }
 
 proc runTester(tester: var Tester, com: CommonRef, testStatusIMPL: var TestStatus) =
-  discard com.db.persistHeaderToDb(tester.genesisHeader, none(DifficultyInt))
+  discard com.db.persistHeaderToDb(tester.genesisHeader,
+    com.consensus == ConsensusType.POS)
   check com.db.getCanonicalHead().blockHash == tester.genesisHeader.blockHash
   let checkSeal = tester.shouldCheckSeal
 
@@ -371,8 +373,9 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = fal
     var success = true
     try:
       tester.runTester(com, testStatusIMPL)
-      let latestBlockHash = com.db.getCanonicalHead().blockHash
+      let header = com.db.getCanonicalHead()
-      if latestBlockHash != tester.lastBlockHash:
+      let lastBlockHash = header.blockHash
+      check lastBlockHash == tester.lastBlockHash
       if tester.postStateHash != Hash256():
         let rootHash = tester.vmState.stateDB.rootHash
         if tester.postStateHash != rootHash:
@@ -380,8 +383,11 @@ proc testFixture(node: JsonNode, testStatusIMPL: var TestStatus, debugMode = fal
             $rootHash & ", get=" &
             $tester.postStateHash
           )
-        else:
+      elif lastBlockHash == tester.lastBlockHash:
-          verifyStateDB(fixture["postState"], tester.vmState.readOnlyStateDB)
+        # multiple chain, we are using the last valid canonical
+        # state root to test against 'postState'
+        let stateDB = AccountsCache.init(memDB, header.stateRoot, pruneTrie)
+        verifyStateDB(fixture["postState"], ReadOnlyStateDB(stateDB))
     except ValidationError as E:
       echo fixtureName, " ERROR: ", E.msg
       success = false

tests/test_rpc.nim

@@ -123,7 +123,8 @@ proc setupEnv(com: CommonRef, signer, ks2: EthAddress, ctx: EthContext): TestEnv
   let uncles = [header]
   header.ommersHash = com.db.persistUncles(uncles)
 
-  discard com.db.persistHeaderToDb(header, none(DifficultyInt))
+  discard com.db.persistHeaderToDb(header,
+    com.consensus == ConsensusType.POS)
   com.db.persistFixtureBlock()
   result = TestEnv(
     txHash: signedTx1.rlpHash,

tests/test_txpool2.nim

@@ -275,13 +275,6 @@ proc runTxHeadDelta*(noisy = true) =
           # Commit to block chain
           check chain.persistBlocks([blk.header], [body]).isOk
 
-          # If not for other reason, setting head is irrelevant for this test
-          #
-          # # PoS block canonical head must be explicitly set using setHead.
-          # # The function `persistHeaderToDb()` used in `persistBlocks()`
-          # # does not reliably do so due to scoring.
-          # chainDB.setHead(blk.header)
-
           # Synchronise TxPool against new chain head, register txs differences.
           # In this particular case, these differences will simply flush the
           # packer bucket.