feat: handle unknown ops and proofs without halting ingestion

The chain contains op types the explorer has no serializer for (e.g.
ChannelSetKeys, opcode 16), and a single such transaction aborted the
entire backfill. Same approach as #19, adapted to the current tx format:
unrecognized opcodes and proof variants (e.g. the NoProof unit variant)
fall back to an UnknownOp/UnknownSignature pair that preserves the raw
data verbatim. Typed support can be added incrementally; known ops keep
their strict proof-type checks.

Also adds a typed ChannelSetKeys serializer (opcode 16), shaped from a
real op captured on the chain (fixture included) — its DB model already
existed, but with List[bytes] keys, which break the JSON column's utf-8
encoding; changed to HexBytes like every other byte field.

src/models/transactions/operations/contents.py

@@ -48,7 +48,9 @@ class ChannelBlob(NbeContent):
 class ChannelSetKeys(NbeContent):
     type: Literal["ChannelSetKeys"] = "ChannelSetKeys"
     channel: HexBytes
-    keys: List[bytes]
+    # HexBytes (not plain bytes): content is stored as JSON in the DB, and raw
+    # bytes break its utf-8 encoding for arbitrary key material.
+    keys: List[HexBytes]
 
 
 class SDPDeclareServiceType(Enum):
@@ -85,6 +87,26 @@ class LeaderClaim(NbeContent):
     mantle_tx_hash: HexBytes
 
 
+class UnknownOp(NbeContent):
+    """Fallback for mantle ops without a typed model (same approach as #19).
+
+    Preserves the opcode and raw payload verbatim so new node op types never
+    break block ingestion; typed support can be added later.
+    """
+
+    type: Literal["UnknownOp"] = "UnknownOp"
+    opcode: int
+    raw_payload: Optional[Any] = None
+
+
 OperationContent = (
-    LedgerTransfer | ChannelInscribe | ChannelBlob | ChannelSetKeys | SDPDeclare | SDPWithdraw | SDPActive | LeaderClaim
+    LedgerTransfer
+    | ChannelInscribe
+    | ChannelBlob
+    | ChannelSetKeys
+    | SDPDeclare
+    | SDPWithdraw
+    | SDPActive
+    | LeaderClaim
+    | UnknownOp
 )

src/models/transactions/operations/proofs.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Literal
+from typing import Any, Literal, Optional
 
 from core.models import NbeSchema
 from core.types import HexBytes
@@ -31,4 +31,15 @@ class ZkAndEd25519Signature(NbeSignature):
     ed25519_signature: HexBytes
 
 
-OperationProof = Ed25519Signature | ZkSignature | ZkAndEd25519Signature
+class UnknownSignature(NbeSignature):
+    """Fallback for proof variants without a typed model (same approach as #19).
+
+    `raw` holds the value verbatim — it covers unit variants like "NoProof"
+    (which carry no signature bytes) as well as future tagged variants.
+    """
+
+    type: Literal["Unknown"] = "Unknown"
+    raw: Optional[Any] = None
+
+
+OperationProof = Ed25519Signature | ZkSignature | ZkAndEd25519Signature | UnknownSignature

src/node/api/serializers/operation.py

@@ -1,3 +1,4 @@
+import logging
 from random import randint
 from typing import Annotated, Any, List, Optional, Self, Union
 
@@ -5,13 +6,16 @@ from pydantic import BeforeValidator, Field
 
 from core.models import NbeSerializer
 from models.transactions.operations.contents import SDPDeclareServiceType
-from node.api.serializers.fields import BytesFromHex, BytesFromIntArray
+from node.api.serializers.fields import BytesFromHex, BytesFromHexOrIntArray
 from node.api.serializers.note import NoteSerializer
 from utils.protocols import FromRandom
 from utils.random import random_bytes
 
+logger = logging.getLogger(__name__)
+
 # Mantle op opcodes.
 OPCODE_LEDGER = 0
+OPCODE_CHANNEL_SET_KEYS = 16
 OPCODE_CHANNEL_INSCRIBE = 17
 OPCODE_SDP_DECLARE = 32
 OPCODE_SDP_ACTIVE = 34
@@ -35,11 +39,29 @@ class LedgerOpSerializer(NbeSerializer, FromRandom):
         )
 
 
+class ChannelSetKeysOpSerializer(NbeSerializer, FromRandom):
+    """Channel set-keys op (opcode 16): sets the signing keys of a channel."""
+
+    channel: BytesFromHexOrIntArray = Field(description="Channel ID.")
+    keys: List[BytesFromHexOrIntArray] = Field(description="Channel signing public keys.")
+
+    @classmethod
+    def from_random(cls) -> Self:
+        return cls.model_validate(
+            {
+                "channel": random_bytes(32).hex(),
+                "keys": [random_bytes(32).hex()],
+            }
+        )
+
+
 class ChannelInscribeOpSerializer(NbeSerializer, FromRandom):
     """Channel inscribe op (opcode 17): writes an inscription to a channel."""
 
     channel_id: BytesFromHex = Field(description="Channel ID in hex format.")
-    inscription: BytesFromIntArray = Field(description="Inscription bytes (int array).")
+    inscription: BytesFromHexOrIntArray = Field(
+        description="Inscription bytes (int array on older nodes, hex string on newer ones)."
+    )
     parent: BytesFromHex = Field(description="Parent inscription hash in hex format.")
     signer: BytesFromHex = Field(description="Signer public key in hex format.")
 
@@ -98,8 +120,20 @@ class SDPActiveOpSerializer(NbeSerializer, FromRandom):
         )
 
 
+class UnknownOpSerializer(NbeSerializer):
+    """Fallback for opcodes without a typed serializer.
+
+    Preserves the opcode and raw payload verbatim so unknown (e.g. newly
+    introduced) op types never break block ingestion.
+    """
+
+    opcode: int
+    payload: Optional[Any] = None
+
+
 OPCODE_TO_SERIALIZER: dict[int, type] = {
     OPCODE_LEDGER: LedgerOpSerializer,
+    OPCODE_CHANNEL_SET_KEYS: ChannelSetKeysOpSerializer,
     OPCODE_CHANNEL_INSCRIBE: ChannelInscribeOpSerializer,
     OPCODE_SDP_DECLARE: SDPDeclareOpSerializer,
     OPCODE_SDP_ACTIVE: SDPActiveOpSerializer,
@@ -108,11 +142,13 @@ OPCODE_TO_SERIALIZER: dict[int, type] = {
 
 MantleOpSerializerVariants = Union[
     LedgerOpSerializer,
+    ChannelSetKeysOpSerializer,
     ChannelInscribeOpSerializer,
     SDPDeclareOpSerializer,
     SDPActiveOpSerializer,
+    UnknownOpSerializer,
 ]
-_MANTLE_OP_SERIALIZER_CLASSES = tuple(OPCODE_TO_SERIALIZER.values())
+_MANTLE_OP_SERIALIZER_CLASSES = tuple(OPCODE_TO_SERIALIZER.values()) + (UnknownOpSerializer,)
 
 
 def _parse_mantle_op(data: Any) -> MantleOpSerializerVariants:
@@ -122,9 +158,8 @@ def _parse_mantle_op(data: Any) -> MantleOpSerializerVariants:
         opcode = data["opcode"]
         serializer_class = OPCODE_TO_SERIALIZER.get(opcode)
         if serializer_class is None:
-            raise ValueError(
-                f"Unsupported mantle op opcode {opcode}; known opcodes: {sorted(OPCODE_TO_SERIALIZER)}."
-            )
+            logger.warning(f"No typed serializer for mantle op opcode {opcode}; storing it verbatim.")
+            return UnknownOpSerializer.model_validate(data)
         return serializer_class.model_validate(data["payload"])
     raise ValueError(f"Cannot parse mantle op from {type(data).__name__}.")
 

src/node/api/serializers/proof.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Annotated, Any, Self, Union
+from typing import Annotated, Any, Optional, Self, Union
 
 from pydantic import BeforeValidator, Field, RootModel
 
@@ -7,10 +7,11 @@ from core.models import NbeSerializer
 from models.transactions.operations.proofs import (
     Ed25519Signature,
     NbeSignature,
+    UnknownSignature,
     ZkAndEd25519Signature,
     ZkSignature,
 )
-from node.api.serializers.fields import BytesFromHex, BytesFromIntArray
+from node.api.serializers.fields import BytesFromHex, BytesFromHexOrIntArray
 from utils.protocols import EnforceSubclassFromRandom
 from utils.random import random_bytes
 
@@ -22,7 +23,7 @@ class OperationProofSerializer(EnforceSubclassFromRandom, ABC):
 
 
 class Ed25519SignatureSerializer(OperationProofSerializer, RootModel[bytes]):
-    root: BytesFromIntArray
+    root: BytesFromHexOrIntArray
 
     def into_operation_proof(self) -> NbeSignature:
         return Ed25519Signature.model_validate(
@@ -39,9 +40,9 @@ class Ed25519SignatureSerializer(OperationProofSerializer, RootModel[bytes]):
 class ZkSignatureSerializer(OperationProofSerializer, NbeSerializer):
     """Groth16 ZK proof: pi_a (32B) + pi_b (64B) + pi_c (32B) = 128 bytes total."""
 
-    pi_a: BytesFromIntArray
-    pi_b: BytesFromIntArray
-    pi_c: BytesFromIntArray
+    pi_a: BytesFromHexOrIntArray
+    pi_b: BytesFromHexOrIntArray
+    pi_c: BytesFromHexOrIntArray
 
     def to_bytes(self) -> bytes:
         return self.pi_a + self.pi_b + self.pi_c
@@ -90,6 +91,23 @@ class ZkAndEd25519SignaturesSerializer(OperationProofSerializer, NbeSerializer):
         )
 
 
+class UnknownProofSerializer(OperationProofSerializer, NbeSerializer):
+    """Fallback for proof variants without a typed serializer (e.g. NoProof).
+
+    Preserves the raw value verbatim so unknown proof types never break block
+    ingestion.
+    """
+
+    raw: Optional[Any] = None
+
+    def into_operation_proof(self) -> NbeSignature:
+        return UnknownSignature.model_validate({"raw": self.raw})
+
+    @classmethod
+    def from_random(cls, *args, **kwargs) -> Self:
+        return cls.model_validate({"raw": "NoProof"})
+
+
 PROOF_TAG_TO_SERIALIZER = {
     "Ed25519Sig": Ed25519SignatureSerializer,
     "ZkSig": ZkSignatureSerializer,
@@ -104,11 +122,13 @@ def _parse_proof(data: Any) -> OperationProofSerializer:
         for tag, serializer_class in PROOF_TAG_TO_SERIALIZER.items():
             if tag in data:
                 return serializer_class.model_validate(data[tag])
-    return data
+    # Unit variants (e.g. "NoProof") arrive as plain strings; unknown tagged
+    # variants arrive as dicts that matched no known tag. Keep them verbatim.
+    return UnknownProofSerializer.model_validate({"raw": data})
 
 
 OperationProofSerializerVariants = Union[
-    Ed25519SignatureSerializer, ZkSignatureSerializer, ZkAndEd25519SignaturesSerializer
+    Ed25519SignatureSerializer, ZkSignatureSerializer, ZkAndEd25519SignaturesSerializer, UnknownProofSerializer
 ]
 OperationProofSerializerField = Annotated[
     OperationProofSerializerVariants,

src/node/api/serializers/signed_transaction.py

@@ -8,13 +8,16 @@ from core.models import NbeSerializer
 from models.transactions.transaction import Transaction
 from node.api.serializers.operation import (
     ChannelInscribeOpSerializer,
+    ChannelSetKeysOpSerializer,
     LedgerOpSerializer,
     SDPActiveOpSerializer,
     SDPDeclareOpSerializer,
+    UnknownOpSerializer,
 )
 from node.api.serializers.proof import (
     Ed25519SignatureSerializer,
     OperationProofSerializerField,
+    UnknownProofSerializer,
     ZkAndEd25519SignaturesSerializer,
     ZkSignatureSerializer,
 )
@@ -33,6 +36,8 @@ def _proof_to_internal(proof) -> dict:
             "zk_signature": proof.zk_signature.to_bytes(),
             "ed25519_signature": proof.ed25519_signature,
         }
+    if isinstance(proof, UnknownProofSerializer):
+        return {"type": "Unknown", "raw": proof.raw}
     raise ValueError(f"Unsupported proof type: {type(proof).__name__}")
 
 
@@ -44,7 +49,12 @@ class SignedTransactionSerializer(NbeSerializer, FromRandom):
     )
 
     def _compute_hash(self) -> bytes:
-        data = self.transaction.model_dump(mode="json")
+        # Prefer the canonical hash reported by the node (newer nodes include it
+        # in mantle_tx). A locally computed JSON hash will NOT match the chain's
+        # real tx hash, so it is only a last-resort fallback for older nodes.
+        if self.transaction.hash is not None:
+            return self.transaction.hash
+        data = self.transaction.model_dump(mode="json", exclude={"hash"})
         canonical = json.dumps(data, sort_keys=True, separators=(",", ":"))
         return hashlib.sha256(canonical.encode()).digest()
 
@@ -116,6 +126,28 @@ class SignedTransactionSerializer(NbeSerializer, FromRandom):
                         "proof": _proof_to_internal(proof),
                     }
                 )
+            elif isinstance(op, ChannelSetKeysOpSerializer):
+                operations.append(
+                    {
+                        "content": {
+                            "type": "ChannelSetKeys",
+                            "channel": op.channel,
+                            "keys": list(op.keys),
+                        },
+                        "proof": _proof_to_internal(proof),
+                    }
+                )
+            elif isinstance(op, UnknownOpSerializer):
+                operations.append(
+                    {
+                        "content": {
+                            "type": "UnknownOp",
+                            "opcode": op.opcode,
+                            "raw_payload": op.payload,
+                        },
+                        "proof": _proof_to_internal(proof),
+                    }
+                )
             else:
                 raise ValueError(f"Unsupported mantle op type: {type(op).__name__}")
 

tests/fixtures/ops_samples_testnet.json

@@ -0,0 +1,78 @@
+{
+  "16": {
+    "payload": {
+      "channel": "d894b6a35644bcb53d9a31be31485b68257368c1d1ee1a0f4a82d85ec63208a5",
+      "keys": [
+        "695c36dc795b084122f596fa7d6abbf35f0f8772d47f01c7ebc294bc0a9d0f8d"
+      ]
+    },
+    "proof": {
+      "Ed25519Sig": [
+        38,
+        229,
+        5,
+        199,
+        203,
+        222,
+        82,
+        178,
+        147,
+        99,
+        199,
+        230,
+        185,
+        102,
+        143,
+        138,
+        88,
+        159,
+        171,
+        28,
+        17,
+        21,
+        8,
+        131,
+        137,
+        164,
+        233,
+        118,
+        100,
+        21,
+        214,
+        227,
+        82,
+        10,
+        71,
+        170,
+        26,
+        50,
+        120,
+        127,
+        230,
+        209,
+        122,
+        65,
+        106,
+        95,
+        138,
+        50,
+        181,
+        121,
+        87,
+        48,
+        57,
+        71,
+        41,
+        0,
+        195,
+        122,
+        165,
+        126,
+        104,
+        137,
+        25,
+        0
+      ]
+    }
+  }
+}

tests/test_node_serializers.py

@@ -0,0 +1,145 @@
+"""Tests for node API serializers against the current (0.1.3-rc) wire format.
+
+The fixtures are real data: block_new_format.json is a block captured from a
+node, and ops_samples_testnet.json holds per-opcode op/proof samples harvested
+from the chain.
+"""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from node.api.http import normalize_info_payload
+from node.api.serializers.block import BlockSerializer
+from node.api.serializers.fields import bytes_from_hex_or_intarray
+from node.api.serializers.info import InfoSerializer
+from node.api.serializers.operation import ChannelSetKeysOpSerializer, UnknownOpSerializer
+from node.api.serializers.proof import Ed25519SignatureSerializer
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+@pytest.fixture
+def block() -> dict:
+    return json.loads((FIXTURES / "block_new_format.json").read_text())
+
+
+class TestInfoNormalization:
+    def test_nested_format(self):
+        payload = {
+            "cryptarchia_info": {"lib": "aa", "lib_slot": 9, "tip": "bb", "slot": 10, "height": 3},
+            "mode": {"Started": "Online"},
+        }
+        info = InfoSerializer.model_validate(normalize_info_payload(payload))
+        assert info.mode == "Online"
+        assert info.height == 3
+        assert info.tip == "bb"
+
+    def test_flat_format_passes_through(self):
+        payload = {"lib": "aa", "tip": "bb", "slot": 1, "height": 2, "mode": "Online"}
+        info = InfoSerializer.model_validate(normalize_info_payload(payload))
+        assert info.mode == "Online"
+        assert info.height == 2
+
+
+class TestHexOrIntArrayField:
+    def test_accepts_hex_string(self):
+        assert bytes_from_hex_or_intarray("68656c6c6f") == b"hello"
+
+    def test_accepts_int_array(self):
+        assert bytes_from_hex_or_intarray([104, 101, 108, 108, 111]) == b"hello"
+
+    def test_rejects_other_types(self):
+        with pytest.raises(ValueError):
+            bytes_from_hex_or_intarray(42)
+
+
+class TestBlockParsing:
+    def test_parses_real_block(self, block):
+        parsed = BlockSerializer.model_validate(block)
+        assert parsed.header.slot == block["header"]["slot"]
+        assert len(parsed.transactions) == len(block["transactions"])
+
+    def test_tx_hash_comes_from_node(self, block):
+        parsed = BlockSerializer.model_validate(block)
+        tx = parsed.transactions[0].into_transaction()
+        expected = bytes.fromhex(block["transactions"][0]["mantle_tx"]["hash"])
+        assert tx.hash == expected
+
+    def test_inscription_decodes_from_hex(self, block):
+        parsed = BlockSerializer.model_validate(block)
+        op = parsed.transactions[0].transaction.ops[0]
+        raw_hex = block["transactions"][0]["mantle_tx"]["ops"][0]["payload"]["inscription"]
+        assert op.inscription == bytes.fromhex(raw_hex)
+        # Sequencer inscriptions reference LEZ program accounts.
+        assert b"/LEZ/" in op.inscription
+
+    def test_ed25519_proof_from_hex(self, block):
+        parsed = BlockSerializer.model_validate(block)
+        proof = parsed.transactions[0].operations_proofs[0]
+        assert isinstance(proof, Ed25519SignatureSerializer)
+        assert len(proof.root) == 64
+
+    def test_into_block_roundtrip(self, block):
+        parsed = BlockSerializer.model_validate(block).into_block()
+        assert parsed.hash == bytes.fromhex(block["header"]["id"])
+        assert parsed.transactions[0].operations[0].content.type == "ChannelInscribe"
+
+    def test_missing_gas_prices_default_to_zero(self, block):
+        parsed = BlockSerializer.model_validate(block)
+        tx = parsed.transactions[0].into_transaction()
+        assert tx.execution_gas_price == 0
+        assert tx.storage_gas_price == 0
+
+    def test_hash_fallback_when_node_omits_it(self, block):
+        del block["transactions"][0]["mantle_tx"]["hash"]
+        parsed = BlockSerializer.model_validate(block)
+        tx = parsed.transactions[0].into_transaction()
+        assert len(tx.hash) == 32  # deterministic local fallback
+
+
+class TestUnknownOps:
+    def test_unknown_opcode_is_preserved_not_fatal(self, block):
+        block["transactions"][0]["mantle_tx"]["ops"][0]["opcode"] = 99
+        parsed = BlockSerializer.model_validate(block)
+        op = parsed.transactions[0].transaction.ops[0]
+        assert isinstance(op, UnknownOpSerializer)
+        assert op.opcode == 99
+        content = parsed.transactions[0].into_transaction().operations[0].content
+        assert content.type == "UnknownOp"
+        assert content.opcode == 99
+        assert content.raw_payload is not None  # raw payload preserved verbatim
+
+    def test_unknown_op_with_noproof_is_preserved_not_fatal(self, block):
+        # e.g. a LeaderClaim carries no proof; neither the op nor the
+        # "NoProof" unit variant should break ingestion.
+        tx = block["transactions"][0]
+        tx["mantle_tx"]["ops"][0] = {"opcode": 48, "payload": {"rewards_root": "aa" * 32}}
+        tx["ops_proofs"][0] = "NoProof"
+        parsed = BlockSerializer.model_validate(block)
+        operation = parsed.transactions[0].into_transaction().operations[0]
+        assert operation.content.type == "UnknownOp"
+        assert operation.content.opcode == 48
+        assert operation.proof.type == "Unknown"
+        assert operation.proof.raw == "NoProof"
+
+
+class TestChannelSetKeysOp:
+    @pytest.fixture
+    def setkeys_sample(self) -> dict:
+        """Real opcode 16 op + proof captured from the chain."""
+        samples = json.loads((FIXTURES / "ops_samples_testnet.json").read_text())
+        return samples["16"]
+
+    def test_real_sample_parses(self, block, setkeys_sample):
+        tx = block["transactions"][0]
+        tx["mantle_tx"]["ops"] = [{"opcode": 16, "payload": setkeys_sample["payload"]}]
+        tx["ops_proofs"] = [setkeys_sample["proof"]]
+        parsed = BlockSerializer.model_validate(block)
+        op = parsed.transactions[0].transaction.ops[0]
+        assert isinstance(op, ChannelSetKeysOpSerializer)
+        assert op.channel == bytes.fromhex(setkeys_sample["payload"]["channel"])
+        assert len(op.keys) == len(setkeys_sample["payload"]["keys"])
+        content = parsed.transactions[0].into_transaction().operations[0].content
+        assert content.type == "ChannelSetKeys"