Sync updates to docs, workloads, and monitoring

Cargo.toml

@@ -37,50 +37,50 @@ testing-framework-runner-local   = { default-features = false, path = "testing-f
 testing-framework-workflows      = { default-features = false, path = "testing-framework/workflows" }
 
 # Nomos git dependencies (pinned to latest master)
-broadcast-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+broadcast-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
 cfgsync = { default-features = false, path = "testing-framework/tools/cfgsync" }
-chain-leader = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8", features = [
+chain-leader = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e", features = [
   "pol-dev-mode",
 ] }
-chain-network = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+chain-network = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-chain-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+chain-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-common-http-client = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+common-http-client = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-cryptarchia-engine = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+cryptarchia-engine = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-cryptarchia-sync = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+cryptarchia-sync = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-executor-http-client = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+executor-http-client = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-groth16 = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+groth16 = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-key-management-system-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+key-management-system-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-kzgrs = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+kzgrs = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-kzgrs-backend = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+kzgrs-backend = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-api = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-api = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-blend-message = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-blend-message = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-blend-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-blend-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-cli = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-cli = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-core = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-core = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-da-dispersal = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-da-dispersal = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-da-network-core = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-da-network-core = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-da-network-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-da-network-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-da-sampling = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-da-sampling = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-da-verifier = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-da-verifier = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-executor = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-executor = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-http-api-common = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-http-api-common = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-ledger = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-ledger = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-libp2p = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-libp2p = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-network = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-network = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-node = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-node = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-sdp = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-sdp = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-time = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-time = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-tracing = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-tracing = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-tracing-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-tracing-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-utils = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-utils = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-nomos-wallet = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+nomos-wallet = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-poc = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+poc = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-pol = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+pol = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-subnetworks-assignations = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+subnetworks-assignations = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-tests = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+tests = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-tx-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+tx-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-wallet = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+wallet = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
-zksign = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+zksign = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "365b36469c34a19c607eed7d8f7a3e5380dac36e" }
 
 # External crates
 async-trait      = { default-features = false, version = "0.1" }
@@ -99,3 +99,45 @@ tempfile         = { default-features = false, version = "3" }
 thiserror        = { default-features = false, version = "2.0" }
 tokio            = { default-features = false, version = "1" }
 tracing          = { default-features = false, version = "0.1" }
+
+[patch."https://github.com/logos-co/nomos-node"]
+broadcast-service = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/chain/broadcast-service" }
+chain-leader = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/chain/chain-leader" }
+chain-network = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/chain/chain-network" }
+chain-service = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/chain/chain-service" }
+common-http-client = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nodes/nomos-node/http-client" }
+cryptarchia-engine = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/consensus/cryptarchia-engine" }
+cryptarchia-sync = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/consensus/cryptarchia-sync" }
+executor-http-client = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nodes/nomos-executor/http-client" }
+groth16 = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/zk/groth16" }
+key-management-system-service = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/key-management-system" }
+kzgrs = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-da/kzgrs" }
+kzgrs-backend = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-da/kzgrs-backend" }
+nomos-api = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/api" }
+nomos-blend-message = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-blend/message" }
+nomos-blend-service = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/blend" }
+nomos-core = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-core/chain-defs" }
+nomos-da-dispersal = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/data-availability/dispersal" }
+nomos-da-network-core = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-da/network/core" }
+nomos-da-network-service = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/data-availability/network" }
+nomos-da-sampling = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/data-availability/sampling" }
+nomos-da-verifier = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/data-availability/verifier" }
+nomos-executor = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nodes/nomos-executor/executor" }
+nomos-http-api-common = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nodes/api-common" }
+nomos-ledger = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/ledger/nomos-ledger" }
+nomos-libp2p = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-libp2p" }
+nomos-network = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/network" }
+nomos-node = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nodes/nomos-node/node" }
+nomos-sdp = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/sdp" }
+nomos-time = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/time" }
+nomos-tracing = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-tracing" }
+nomos-tracing-service = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/tracing" }
+nomos-utils = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-utils" }
+nomos-wallet = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/wallet" }
+poc = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/zk/proofs/poc" }
+pol = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/zk/proofs/pol" }
+subnetworks-assignations = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-da/network/subnetworks-assignations" }
+tests = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/tests" }
+tx-service = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/nomos-services/tx-service" }
+wallet = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/wallet" }
+zksign = { path = "/Users/andrussalumets/IdeaProjects/nomos-node/zk/proofs/zksign" }

book/src/dsl-cheat-sheet.md

@@ -42,8 +42,9 @@ ScenarioBuilder::topology_with(|t| {
 
 ```rust
 .da_with(|da| {
-    da.channel_rate(1)       // 1 channel operation per block
+    da.channel_rate(1)       // number of DA channels to run
-        .blob_rate(2)        // 2 blob dispersals per block
+        .blob_rate(2)        // target 2 blobs per block (headroom applied)
+        .headroom_percent(20)// optional headroom when sizing channels
 })                           // Finish DA workload config
 ```
 
@@ -118,8 +119,9 @@ async fn run_test() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
                 .users(20)
         })
         .da_with(|da| {
-            da.channel_rate(1)             // 1 channel operation per block
+            da.channel_rate(1)             // number of DA channels
-                .blob_rate(2)              // 2 blob dispersals per block
+                .blob_rate(2)              // target 2 blobs per block
+                .headroom_percent(20)      // optional channel headroom
         })
         .expect_consensus_liveness()
         .with_run_duration(Duration::from_secs(90))

book/src/examples.md

@@ -103,7 +103,7 @@ async fn da_and_transactions() -> Result<(), Box<dyn std::error::Error + Send +
                 .users(15)
         })
         .da_with(|da| {
-            da.channel_rate(1)
+            da.channel_rate(2)
                 .blob_rate(2)
         })
         .expect_consensus_liveness()

book/src/quickstart.md

@@ -48,14 +48,15 @@ let mut plan = ScenarioBuilder::topology_with(|t| {
             .validators(1)
             .executors(1)
     })
-    .wallets(64)
+    .wallets(1_000)
     .transactions_with(|txs| {
         txs.rate(5)                 // 5 transactions per block
-            .users(8)
+            .users(500)             // use 500 of the seeded wallets
     })
     .da_with(|da| {
-        da.channel_rate(1)         // 1 channel operation per block
+        da.channel_rate(1)          // 1 channel
-            .blob_rate(1)          // 1 blob dispersal per block
+            .blob_rate(1)           // target 1 blob per block
+            .headroom_percent(20)   // default headroom when sizing channels
     })
     .expect_consensus_liveness()
     .with_run_duration(Duration::from_secs(60))
@@ -97,7 +98,7 @@ This defines **what** your test network looks like.
 ### 2. Wallet Seeding
 
 ```rust
-.wallets(64)                 // Seed 64 funded wallet accounts
+.wallets(1_000)              // Seed 1,000 funded wallet accounts
 ```
 
 Provides funded accounts for transaction submission.
@@ -107,11 +108,12 @@ Provides funded accounts for transaction submission.
 ```rust
 .transactions_with(|txs| {
     txs.rate(5)              // 5 transactions per block
-        .users(8)            // Use 8 of the 64 wallets
+        .users(500)          // Use 500 of the 1,000 wallets
 })
 .da_with(|da| {
-    da.channel_rate(1)       // 1 channel operation per block
+    da.channel_rate(1)       // 1 DA channel (more spawned with headroom)
-        .blob_rate(1)        // 1 blob dispersal per block
+        .blob_rate(1)        // target 1 blob per block
+        .headroom_percent(20)// default headroom when sizing channels
 })
 ```
 

examples/src/bin/compose_runner.rs

@@ -9,8 +9,8 @@ const DEFAULT_VALIDATORS: usize = 1;
 const DEFAULT_EXECUTORS: usize = 1;
 const DEFAULT_RUN_SECS: u64 = 60;
 const MIXED_TXS_PER_BLOCK: u64 = 5;
-const TOTAL_WALLETS: usize = 64;
+const TOTAL_WALLETS: usize = 1000;
-const TRANSACTION_WALLETS: usize = 8;
+const TRANSACTION_WALLETS: usize = 500;
 
 #[tokio::main]
 async fn main() {

examples/src/bin/k8s_runner.rs

@@ -9,8 +9,8 @@ const DEFAULT_RUN_SECS: u64 = 60;
 const DEFAULT_VALIDATORS: usize = 1;
 const DEFAULT_EXECUTORS: usize = 1;
 const MIXED_TXS_PER_BLOCK: u64 = 5;
-const TOTAL_WALLETS: usize = 64;
+const TOTAL_WALLETS: usize = 1000;
-const TRANSACTION_WALLETS: usize = 8;
+const TRANSACTION_WALLETS: usize = 500;
 
 #[tokio::main]
 async fn main() {
@@ -59,8 +59,7 @@ async fn run_k8s_case(
                 .users(TRANSACTION_WALLETS)
         })
         .da_with(|da| {
-            da.channel_rate(1)
+            da.blob_rate(1)
-                .blob_rate(1)
         })
         .with_run_duration(run_duration)
         .expect_consensus_liveness()

examples/src/bin/local_runner.rs

@@ -9,8 +9,8 @@ const DEFAULT_VALIDATORS: usize = 1;
 const DEFAULT_EXECUTORS: usize = 1;
 const DEFAULT_RUN_SECS: u64 = 60;
 const MIXED_TXS_PER_BLOCK: u64 = 5;
-const TOTAL_WALLETS: usize = 64;
+const TOTAL_WALLETS: usize = 1000;
-const TRANSACTION_WALLETS: usize = 8;
+const TRANSACTION_WALLETS: usize = 500;
 
 #[tokio::main]
 async fn main() {
@@ -62,7 +62,7 @@ async fn run_local_case(
     })
     .wallets(TOTAL_WALLETS)
     .transactions_with(|txs| txs.rate(MIXED_TXS_PER_BLOCK).users(TRANSACTION_WALLETS))
-    .da_with(|da| da.channel_rate(1).blob_rate(1))
+    .da_with(|da| da.blob_rate(1))
     .with_run_duration(run_duration)
     .expect_consensus_liveness()
     .build();

scripts/build-bundle.sh

@@ -3,9 +3,17 @@ set -euo pipefail
 
 # Build a nomos-binaries.tar.gz for the specified platform.
 #
-# Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH]
+# Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH] [--rev REV | --path DIR] [--features LIST]
 #   --platform   Target platform for binaries (default: host)
 #   --output     Output path for the tarball (default: .tmp/nomos-binaries-<platform>-<version>.tar.gz)
+#   --rev        nomos-node git revision to build (overrides NOMOS_NODE_REV)
+#   --path       Use local nomos-node checkout at DIR (skip fetch/checkout)
+#   --features   Extra cargo features to enable (comma-separated); base always includes "testing"
+
+# Always run under bash; bail out if someone invokes via sh.
+if [ -z "${BASH_VERSION:-}" ]; then
+  exec bash "$0" "$@"
+fi
 
 usage() {
   cat <<'EOF'
@@ -14,11 +22,14 @@ Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH]
 Options:
   --platform   Target platform for binaries (default: host)
   --output     Output path for the tarball (default: .tmp/nomos-binaries-<platform>-<version>.tar.gz)
+  --rev        nomos-node git revision to build (overrides NOMOS_NODE_REV)
+  --path       Use local nomos-node checkout at DIR (skip fetch/checkout)
+  --features   Extra cargo features to enable (comma-separated); base always includes "testing"
 
 Notes:
   - For compose/k8s, use platform=linux. If running on macOS, this script will
     run inside a Linux Docker container to produce Linux binaries.
-  - VERSION and NOMOS_NODE_REV env vars are honored (defaults align with run-examples.sh).
+  - VERSION, NOMOS_NODE_REV, and optional NOMOS_NODE_PATH env vars are honored (defaults align with run-examples.sh).
 EOF
 }
 
@@ -37,9 +48,16 @@ else
   exit 1
 fi
 DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}"
-DEFAULT_NODE_REV="${NOMOS_NODE_REV:?Missing NOMOS_NODE_REV in versions.env}"
+DEFAULT_NODE_REV="${NOMOS_NODE_REV:-}"
+DEFAULT_NODE_PATH="${NOMOS_NODE_PATH:-}"
 PLATFORM="host"
 OUTPUT=""
+REV_OVERRIDE=""
+PATH_OVERRIDE=""
+
+# To avoid confusing cache corruption errors inside the Dockerized Linux build,
+# always start from a clean cargo registry/git cache for the cross-build.
+rm -rf "${ROOT_DIR}/.tmp/cargo-linux/registry" "${ROOT_DIR}/.tmp/cargo-linux/git"
 
 while [ "$#" -gt 0 ]; do
   case "$1" in
@@ -47,6 +65,12 @@ while [ "$#" -gt 0 ]; do
       PLATFORM="${2:-}"; shift 2 ;;
     --output|-o)
       OUTPUT="${2:-}"; shift 2 ;;
+    --rev)
+      REV_OVERRIDE="${2:-}"; shift 2 ;;
+    --path)
+      PATH_OVERRIDE="${2:-}"; shift 2 ;;
+    --features)
+      NOMOS_EXTRA_FEATURES="${2:-}"; shift 2 ;;
     *) fail "Unknown argument: $1" ;;
   esac
 done
@@ -57,7 +81,14 @@ case "$PLATFORM" in
 esac
 
 VERSION="${DEFAULT_VERSION}"
-NOMOS_NODE_REV="${DEFAULT_NODE_REV}"
+if [ -n "${REV_OVERRIDE}" ] && [ -n "${PATH_OVERRIDE}" ]; then
+  fail "Use either --rev or --path, not both"
+fi
+if [ -z "${REV_OVERRIDE}" ] && [ -z "${PATH_OVERRIDE}" ] && [ -z "${DEFAULT_NODE_REV}" ] && [ -z "${DEFAULT_NODE_PATH}" ]; then
+  fail "Provide --rev, --path, or set NOMOS_NODE_REV/NOMOS_NODE_PATH in versions.env"
+fi
+NOMOS_NODE_REV="${REV_OVERRIDE:-${DEFAULT_NODE_REV}}"
+NOMOS_NODE_PATH="${PATH_OVERRIDE:-${DEFAULT_NODE_PATH}}"
 
 # Normalize OUTPUT to an absolute path under the workspace.
 if [ -z "${OUTPUT}" ]; then
@@ -72,60 +103,114 @@ if [ "$PLATFORM" = "linux" ] && [ "$(uname -s)" != "Linux" ] && [ -z "${BUNDLE_I
   if ! command -v docker >/dev/null 2>&1; then
     fail "Docker is required to build a Linux bundle from non-Linux host"
   fi
+  NODE_PATH_ENV="${NOMOS_NODE_PATH}"
+  EXTRA_MOUNTS=()
+  if [ -n "${NOMOS_NODE_PATH}" ]; then
+    case "${NOMOS_NODE_PATH}" in
+      "${ROOT_DIR}"/*)
+        NODE_PATH_ENV="/workspace${NOMOS_NODE_PATH#"${ROOT_DIR}"}"
+        ;;
+      /*)
+        NODE_PATH_ENV="/external/nomos-node"
+        EXTRA_MOUNTS+=("-v" "${NOMOS_NODE_PATH}:${NODE_PATH_ENV}")
+        ;;
+      *)
+        fail "--path must be absolute when cross-building in Docker"
+        ;;
+    esac
+  fi
   echo "==> Building Linux bundle inside Docker"
   # Map host OUTPUT path into container.
   container_output="/workspace${OUTPUT#"${ROOT_DIR}"}"
   mkdir -p "${ROOT_DIR}/.tmp/cargo-linux" "${ROOT_DIR}/.tmp/nomos-node-linux-target"
+  FEATURES_ARGS=()
+  if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then
+    # Forward the outer --features flag into the inner Dockerized build
+    FEATURES_ARGS+=(--features "${NOMOS_EXTRA_FEATURES}")
+  fi
   docker run --rm \
     -e VERSION="$VERSION" \
     -e NOMOS_NODE_REV="$NOMOS_NODE_REV" \
+    -e NOMOS_NODE_PATH="$NODE_PATH_ENV" \
     -e NOMOS_CIRCUITS="/workspace/.tmp/nomos-circuits-linux" \
     -e STACK_DIR="/workspace/.tmp/nomos-circuits-linux" \
     -e HOST_DIR="/workspace/.tmp/nomos-circuits-linux" \
+    -e NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}" \
     -e BUNDLE_IN_CONTAINER=1 \
     -e CARGO_HOME=/workspace/.tmp/cargo-linux \
     -e CARGO_TARGET_DIR=/workspace/.tmp/nomos-node-linux-target \
     -v "${ROOT_DIR}/.tmp/cargo-linux":/workspace/.tmp/cargo-linux \
     -v "${ROOT_DIR}/.tmp/nomos-node-linux-target":/workspace/.tmp/nomos-node-linux-target \
     -v "$ROOT_DIR":/workspace \
+    "${EXTRA_MOUNTS[@]}" \
     -w /workspace \
     rust:1.80-bullseye \
-    bash -c "apt-get update && apt-get install -y clang llvm-dev libclang-dev pkg-config cmake libssl-dev rsync libgmp10 libgmp-dev libgomp1 nasm && ./scripts/build-bundle.sh --platform linux --output \"${container_output}\""
+    bash -c "apt-get update && apt-get install -y clang llvm-dev libclang-dev pkg-config cmake libssl-dev rsync libgmp10 libgmp-dev libgomp1 nasm && ./scripts/build-bundle.sh --platform linux --path \"${NODE_PATH_ENV}\" --output \"${container_output}\" ${FEATURES_ARGS[*]}"
   exit 0
 fi
 
 echo "==> Preparing circuits (version ${VERSION})"
 if [ "$PLATFORM" = "host" ]; then
   CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-host"
-  NODE_SRC="${ROOT_DIR}/.tmp/nomos-node-host-src"
+  NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-host-src"
   NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-host-target"
 else
   CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-linux"
-  NODE_SRC="${ROOT_DIR}/.tmp/nomos-node-linux-src"
+  NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-linux-src"
   NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-linux-target"
 fi
+NODE_SRC="${NOMOS_NODE_PATH:-${NODE_SRC_DEFAULT}}"
+DEFAULT_NODE_TARGET="${NODE_TARGET}"
+if [ -n "${NOMOS_NODE_PATH}" ]; then
+  # Avoid mixing stale cloned sources/targets when using a local checkout.
+  rm -rf "${NODE_SRC_DEFAULT}"
+  if [ -d "${NODE_TARGET}" ]; then
+    # Target dir may be a mounted volume; avoid removing the mount point itself.
+    find "${NODE_TARGET}" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+  fi
+  NODE_TARGET="${NODE_TARGET}-local"
+fi
+echo "Using nomos-node source: ${NODE_SRC}"
 export NOMOS_CIRCUITS="${CIRCUITS_DIR}"
 mkdir -p "${ROOT_DIR}/.tmp" "${CIRCUITS_DIR}"
-STACK_DIR="${CIRCUITS_DIR}" HOST_DIR="${CIRCUITS_DIR}" \
+if [ -f "${CIRCUITS_DIR}/${KZG_FILE:-kzgrs_test_params}" ]; then
-  "${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}" </dev/null
+  echo "Circuits already present at ${CIRCUITS_DIR}; skipping download"
+else
+  STACK_DIR="${CIRCUITS_DIR}" HOST_DIR="${CIRCUITS_DIR}" \
+    "${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}" </dev/null
+fi
 
 NODE_BIN="${NODE_TARGET}/debug/nomos-node"
 EXEC_BIN="${NODE_TARGET}/debug/nomos-executor"
 CLI_BIN="${NODE_TARGET}/debug/nomos-cli"
+FEATURES="testing"
+if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then
+  FEATURES="${FEATURES},${NOMOS_EXTRA_FEATURES}"
+fi
 
 echo "==> Building host binaries (platform=${PLATFORM})"
 mkdir -p "${NODE_SRC}"
-if [ ! -d "${NODE_SRC}/.git" ]; then
+if [ -n "${NOMOS_NODE_PATH}" ]; then
-  git clone https://github.com/logos-co/nomos-node.git "${NODE_SRC}"
+  if [ ! -d "${NODE_SRC}" ]; then
+    fail "NOMOS_NODE_PATH does not exist: ${NODE_SRC}"
+  fi
+  echo "Using local nomos-node checkout at ${NODE_SRC} (no fetch/checkout)"
+else
+  if [ ! -d "${NODE_SRC}/.git" ]; then
+    git clone https://github.com/logos-co/nomos-node.git "${NODE_SRC}"
+  fi
+  (
+    cd "${NODE_SRC}"
+    git fetch --depth 1 origin "${NOMOS_NODE_REV}"
+    git checkout "${NOMOS_NODE_REV}"
+    git reset --hard
+    git clean -fdx
+  )
 fi
-(
+  (
-  cd "${NODE_SRC}"
+    cd "${NODE_SRC}"
-  git fetch --depth 1 origin "${NOMOS_NODE_REV}"
-  git checkout "${NOMOS_NODE_REV}"
-  git reset --hard
-  git clean -fdx
   RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \
-    cargo build --features testing \
+    cargo build --features "${FEATURES}" \
     -p nomos-node -p nomos-executor -p nomos-cli \
     --target-dir "${NODE_TARGET}"
 )
@@ -149,3 +234,10 @@ else
   tar -czf "${OUTPUT}" -C "${bundle_dir}" artifacts
 fi
 echo "Bundle created at ${OUTPUT}"
+if [[ "${FEATURES}" == *profiling* ]]; then
+  cat <<'EOF'
+Profiling endpoints (enabled by --features profiling):
+  CPU pprof (SVG):   curl "http://<node-host>:8722/debug/pprof/profile?seconds=15&format=svg" -o profile.svg
+  CPU pprof (proto): go tool pprof -http=:8080 "http://<node-host>:8722/debug/pprof/profile?seconds=15&format=proto"
+EOF
+fi

scripts/query-prom-metrics.sh

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Query all metric names from a Prometheus endpoint and print one sample value
+# per metric (if present).
+#
+# Usage:
+#   PROM_URL=http://127.0.0.1:9090 ./scripts/query-prom-metrics.sh
+#   ./scripts/query-prom-metrics.sh http://127.0.0.1:59804
+
+PROM_URL="${1:-${PROM_URL:-http://127.0.0.1:9090}}"
+
+require() { command -v "$1" >/dev/null 2>&1 || { echo "$1 is required but not installed" >&2; exit 1; }; }
+require jq
+require python3
+
+echo "Querying Prometheus at ${PROM_URL}"
+python3 - <<'PY'
+import os, sys, json, urllib.parse, urllib.request
+
+prom = os.environ.get("PROM_URL")
+if not prom:
+    sys.exit("PROM_URL is not set")
+
+def fetch(path, params=None):
+    url = prom + path
+    if params:
+        url += "?" + urllib.parse.urlencode(params)
+    with urllib.request.urlopen(url, timeout=10) as resp:
+        return json.load(resp)
+
+names = fetch("/api/v1/label/__name__/values").get("data", [])
+if not names:
+    sys.exit("No metrics found or failed to reach Prometheus")
+
+jobs = fetch("/api/v1/label/job/values").get("data", [])
+if jobs:
+    print("Jobs seen:", ", ".join(sorted(jobs)))
+else:
+    print("Jobs seen: <none>")
+
+by_job = {j: [] for j in jobs} if jobs else {}
+
+for name in sorted(names):
+    data = fetch("/api/v1/query", {"query": name}).get("data", {}).get("result", [])
+    for series in data:
+        labels = series.get("metric", {})
+        value = series.get("value", ["", "N/A"])[1]
+        job = labels.get("job", "<no-job>")
+        by_job.setdefault(job, []).append((name, value))
+
+if not by_job:
+    sys.exit("No metric samples returned")
+
+for job in sorted(by_job):
+    print(f"{job}:")
+    samples = by_job[job]
+    if not samples:
+        print("  <no samples>")
+    else:
+        for name, value in sorted(samples):
+            print(f"  {name}: {value}")
+PY

scripts/run-examples.sh

@@ -15,7 +15,7 @@ set -euo pipefail
 #   NOMOS_TESTNET_IMAGE           - image tag (default logos-blockchain-testing:local)
 #   NOMOS_CIRCUITS_PLATFORM       - override host platform detection
 #   NOMOS_CIRCUITS_REBUILD_RAPIDSNARK - set to 1 to force rapidsnark rebuild
-#   NOMOS_NODE_REV                - nomos-node git rev for local binaries (default d2dd5a5084e1daef4032562c77d41de5e4d495f8)
+#   NOMOS_BINARIES_TAR            - path to prebuilt binaries/circuits tarball (required; default .tmp/nomos-binaries-<mode>-<version>.tar.gz)
 
 usage() {
   cat <<'EOF'
@@ -36,8 +36,7 @@ Environment:
   NOMOS_TESTNET_IMAGE            Image tag (default logos-blockchain-testing:local)
   NOMOS_CIRCUITS_PLATFORM        Override host platform detection
   NOMOS_CIRCUITS_REBUILD_RAPIDSNARK  Force rapidsnark rebuild
-  NOMOS_NODE_REV                 nomos-node git rev (default d2dd5a5084e1daef4032562c77d41de5e4d495f8)
+  NOMOS_BINARIES_TAR             Path to prebuilt binaries/circuits tarball (required)
-  NOMOS_BINARIES_TAR             Path to prebuilt binaries/circuits tarball
   NOMOS_SKIP_IMAGE_BUILD         Set to 1 to skip rebuilding the compose/k8s image
 EOF
 }
@@ -70,19 +69,15 @@ if [ -f "${ROOT_DIR}/paths.env" ]; then
   . "${ROOT_DIR}/paths.env"
 fi
 readonly DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}"
-readonly DEFAULT_NODE_REV="${NOMOS_NODE_REV:?Missing NOMOS_NODE_REV in versions.env}"
 readonly KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}"
 readonly KZG_FILE="${NOMOS_KZG_FILE:-kzgrs_test_params}"
 readonly KZG_CONTAINER_PATH="${NOMOS_KZG_CONTAINER_PATH:-/kzgrs_test_params/kzgrs_test_params}"
 readonly HOST_KZG_DIR="${ROOT_DIR}/${KZG_DIR_REL}"
 readonly HOST_KZG_FILE="${HOST_KZG_DIR}/${KZG_FILE}"
-readonly HOST_CIRCUITS_DIR="${ROOT_DIR}/${NOMOS_CIRCUITS_HOST_DIR_REL:-.tmp/nomos-circuits-host}"
-readonly LINUX_CIRCUITS_DIR="${ROOT_DIR}/${NOMOS_CIRCUITS_LINUX_DIR_REL:-.tmp/nomos-circuits-linux}"
 MODE="compose"
 RUN_SECS_RAW=""
 VERSION="${DEFAULT_VERSION}"
 IMAGE="${NOMOS_TESTNET_IMAGE:-logos-blockchain-testing:local}"
-NOMOS_NODE_REV="${DEFAULT_NODE_REV}"
 DEMO_VALIDATORS=""
 DEMO_EXECUTORS=""
 while [ "$#" -gt 0 ]; do
@@ -145,15 +140,28 @@ default_tar_path() {
     return
   fi
   case "$MODE" in
-    host) echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" ;;
+    host)
-    compose|k8s) echo "${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz" ;;
+      echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz"
+      ;;
+    compose|k8s)
+      # When skipping image rebuild, we need host-arch tools (witness generators) on the runner.
+      if [ "${NOMOS_SKIP_IMAGE_BUILD:-}" = "1" ]; then
+        echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz"
+      else
+        echo "${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
+      fi
+      ;;
     *) echo "${ROOT_DIR}/.tmp/nomos-binaries-${VERSION}.tar.gz" ;;
   esac
 }
 
 restore_binaries_from_tar() {
   local tar_path
-  tar_path="$(default_tar_path)"
+  if [ -n "${_RESTORE_TAR_OVERRIDE:-}" ]; then
+    tar_path="${_RESTORE_TAR_OVERRIDE}"
+  else
+    tar_path="$(default_tar_path)"
+  fi
   local extract_dir="${ROOT_DIR}/.tmp/nomos-binaries"
   if [ ! -f "$tar_path" ]; then
     return 1
@@ -169,11 +177,21 @@ restore_binaries_from_tar() {
   local bin_dst="${ROOT_DIR}/testing-framework/assets/stack/bin"
   local circuits_src="${src}/circuits"
   local circuits_dst="${HOST_KZG_DIR}"
+  RESTORED_BIN_DIR="${src}"
+  export RESTORED_BIN_DIR
   if [ -f "${src}/nomos-node" ] && [ -f "${src}/nomos-executor" ] && [ -f "${src}/nomos-cli" ]; then
-    mkdir -p "${bin_dst}"
+    local copy_bins=1
-    cp "${src}/nomos-node" "${src}/nomos-executor" "${src}/nomos-cli" "${bin_dst}/"
+    if [ "$MODE" != "host" ] && ! host_bin_matches_arch "${src}/nomos-node"; then
+      echo "Bundled binaries do not match host arch; skipping copy so containers rebuild from source."
+      copy_bins=0
+      rm -f "${bin_dst}/nomos-node" "${bin_dst}/nomos-executor" "${bin_dst}/nomos-cli"
+    fi
+    if [ "$copy_bins" -eq 1 ]; then
+      mkdir -p "${bin_dst}"
+      cp "${src}/nomos-node" "${src}/nomos-executor" "${src}/nomos-cli" "${bin_dst}/"
+    fi
   else
-    echo "Binaries missing in ${tar_path}; fallback to build-from-source path (run build-binaries workflow to populate)" >&2
+    echo "Binaries missing in ${tar_path}; provide a prebuilt binaries tarball." >&2
     return 1
   fi
   if [ -d "${circuits_src}" ] && [ -f "${circuits_src}/${KZG_FILE}" ]; then
@@ -186,7 +204,7 @@ restore_binaries_from_tar() {
       cp -a "${circuits_src}/." "${circuits_dst}/"
     fi
   else
-    echo "Circuits missing in ${tar_path}; fallback to download/build path (run build-binaries workflow to populate)" >&2
+    echo "Circuits missing in ${tar_path}; provide a prebuilt binaries/circuits tarball." >&2
     return 1
   fi
   RESTORED_BINARIES=1
@@ -211,107 +229,29 @@ host_bin_matches_arch() {
   return 1
 }
 
-ensure_host_binaries() {
+HOST_TAR="${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz"
-  # Build nomos-node/nomos-executor for the host if not already present.
+LINUX_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
-  HOST_SRC="${ROOT_DIR}/.tmp/nomos-node-host-src"
+NEED_HOST_RESTORE_AFTER_IMAGE=0
-  HOST_TARGET="${ROOT_DIR}/.tmp/nomos-node-host-target"
-  HOST_NODE_BIN_DEFAULT="${HOST_TARGET}/debug/nomos-node"
-  HOST_EXEC_BIN_DEFAULT="${HOST_TARGET}/debug/nomos-executor"
-  HOST_ASSET_NODE_BIN="${ROOT_DIR}/testing-framework/assets/stack/bin/nomos-node"
-  HOST_ASSET_EXEC_BIN="${ROOT_DIR}/testing-framework/assets/stack/bin/nomos-executor"
 
-  if [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN}" ] && [ -x "${NOMOS_EXECUTOR_BIN:-}" ]; then
+if [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN}" ]; then
-    if host_bin_matches_arch "${NOMOS_NODE_BIN}"; then
+  echo "==> Using pre-specified host binaries (NOMOS_NODE_BIN/NOMOS_EXECUTOR_BIN); skipping tarball restore"
-      echo "Using provided host binaries:"
+else
-      echo "  NOMOS_NODE_BIN=${NOMOS_NODE_BIN}"
+  # On non-Linux compose/k8s runs, use the Linux bundle for image build, then restore host bundle for the runner.
-      echo "  NOMOS_EXECUTOR_BIN=${NOMOS_EXECUTOR_BIN}"
+  if [ "$MODE" != "host" ] && [ "$(uname -s)" != "Linux" ] && [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "0" ] && [ -f "${LINUX_TAR}" ]; then
-      return
+    NEED_HOST_RESTORE_AFTER_IMAGE=1
-    else
+    _RESTORE_TAR_OVERRIDE="${LINUX_TAR}" restore_binaries_from_tar || true
-      echo "Provided NOMOS_NODE_BIN does not match host arch; rebuilding..."
+    unset _RESTORE_TAR_OVERRIDE
-    fi
   fi
 
-  if host_bin_matches_arch "${HOST_NODE_BIN_DEFAULT}" && host_bin_matches_arch "${HOST_EXEC_BIN_DEFAULT}"; then
+  if ! restore_binaries_from_tar; then
-    echo "Host binaries already built at ${HOST_TARGET}"
+    echo "ERROR: Missing or invalid binaries tarball. Provide it via NOMOS_BINARIES_TAR or place it at $(default_tar_path)." >&2
-    NOMOS_NODE_BIN="${HOST_NODE_BIN_DEFAULT}"
+    exit 1
-    NOMOS_EXECUTOR_BIN="${HOST_EXEC_BIN_DEFAULT}"
-    export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN
-    return
   fi
+fi
 
-  if [ "${RESTORED_BINARIES}" -eq 1 ] && host_bin_matches_arch "${HOST_ASSET_NODE_BIN}" && host_bin_matches_arch "${HOST_ASSET_EXEC_BIN}"; then
+echo "==> Using restored circuits/binaries bundle"
-    echo "Using restored host binaries from bundle"
-    NOMOS_NODE_BIN="${HOST_ASSET_NODE_BIN}"
-    NOMOS_EXECUTOR_BIN="${HOST_ASSET_EXEC_BIN}"
-    export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN
-    return
-  fi
-
-  echo "Building host nomos-node/nomos-executor from ${NOMOS_NODE_REV}"
-  mkdir -p "${HOST_SRC}"
-  if [ ! -d "${HOST_SRC}/.git" ]; then
-    git clone https://github.com/logos-co/nomos-node.git "${HOST_SRC}"
-  fi
-  (
-    cd "${HOST_SRC}"
-    git fetch --depth 1 origin "${NOMOS_NODE_REV}"
-    git checkout "${NOMOS_NODE_REV}"
-    git reset --hard
-    git clean -fdx
-    echo "-> Compiling host binaries (may take a few minutes)..."
-    RUSTFLAGS='--cfg feature="pol-dev-mode"' \
-      NOMOS_CIRCUITS="${HOST_BUNDLE_PATH}" \
-      cargo build --features "testing" \
-        -p nomos-node -p nomos-executor -p nomos-cli \
-        --target-dir "${HOST_TARGET}"
-  )
-  NOMOS_NODE_BIN="${HOST_NODE_BIN_DEFAULT}"
-  NOMOS_EXECUTOR_BIN="${HOST_EXEC_BIN_DEFAULT}"
-  export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN
-}
-
-restore_binaries_from_tar || true
-
-echo "==> Preparing circuits (version ${VERSION})"
 SETUP_OUT="$(mktemp -t nomos-setup-output.XXXXXX)"
-if [ "${RESTORED_BINARIES}" -ne 1 ]; then
-  "${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}" </dev/null | tee "$SETUP_OUT"
-else
-  echo "Skipping circuits setup; using restored bundle"
-fi
-
-# When running compose/k8s on macOS, prefer host-installed circuits so the
-# host-side zksign tooling matches the host architecture even if the bundle was
-# restored from a linux tarball.
-if [ "${RESTORED_BINARIES}" -eq 1 ] && [ "$MODE" != "host" ] && [ "$(uname -s)" != "Linux" ]; then
-  "${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}" </dev/null | tee -a "$SETUP_OUT"
-fi
-
-# Prefer host-native bundle for host tooling when available; otherwise fall back
-# to the restored circuits location.
-if [ "$(uname -s)" != "Linux" ] && [ -d "${HOST_CIRCUITS_DIR}" ]; then
-  HOST_BUNDLE_PATH="${HOST_CIRCUITS_DIR}"
-elif [ "${RESTORED_BINARIES}" -eq 1 ]; then
-  HOST_BUNDLE_PATH="${HOST_KZG_DIR}"
-else
-  HOST_BUNDLE_PATH="${HOST_KZG_DIR}"
-fi
-
-# If the host bundle was somehow pruned, repair it once more.
-if [ ! -x "${HOST_BUNDLE_PATH}/zksign/witness_generator" ]; then
-  echo "Host circuits missing zksign/witness_generator; repairing..."
-  "${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}"
-fi
-KZG_HOST_PATH="${HOST_BUNDLE_PATH}/${KZG_FILE}"
-if [ ! -f "${KZG_HOST_PATH}" ]; then
-  echo "KZG params missing at ${KZG_HOST_PATH}; rebuilding circuits bundle"
-  "${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}"
-fi
-
 if [ "$MODE" != "host" ]; then
-  if [ "${RESTORED_BINARIES}" -ne 1 ]; then
-    echo "WARNING: NOMOS_BINARIES_TAR not restored; compose/k8s will rebuild binaries from source" >&2
-  fi
   if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then
     echo "==> Skipping testnet image rebuild (NOMOS_SKIP_IMAGE_BUILD=1)"
   else
@@ -321,21 +261,52 @@ if [ "$MODE" != "host" ]; then
   fi
 fi
 
-if [ "$MODE" = "host" ]; then
+if [ "${NEED_HOST_RESTORE_AFTER_IMAGE}" = "1" ]; then
-  if [ "${RESTORED_BINARIES}" -eq 1 ] && [ "$(uname -s)" = "Linux" ]; then
+  if [ -f "${HOST_TAR}" ]; then
-    tar_node="${ROOT_DIR}/testing-framework/assets/stack/bin/nomos-node"
+    echo "==> Restoring host bundle for runner (${HOST_TAR})"
-    tar_exec="${ROOT_DIR}/testing-framework/assets/stack/bin/nomos-executor"
+    _RESTORE_TAR_OVERRIDE="${HOST_TAR}" restore_binaries_from_tar || {
-    if [ -x "${tar_node}" ] && [ -x "${tar_exec}" ]; then
+      echo "ERROR: Failed to restore host bundle from ${HOST_TAR}" >&2
-      echo "==> Using restored host binaries from tarball"
+      exit 1
-      NOMOS_NODE_BIN="${tar_node}"
+    }
-      NOMOS_EXECUTOR_BIN="${tar_exec}"
+    unset _RESTORE_TAR_OVERRIDE
-      export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN
+    echo "==> Using restored circuits/binaries bundle"
-    else
-      echo "Restored tarball missing executables for host; building host binaries..."
-      ensure_host_binaries
-    fi
   else
-    ensure_host_binaries
+    echo "ERROR: Expected host bundle at ${HOST_TAR} for runner." >&2
+    exit 1
+  fi
+fi
+
+HOST_BUNDLE_PATH="${HOST_KZG_DIR}"
+
+# If the host bundle was somehow pruned, repair it once more.
+if [ ! -x "${HOST_BUNDLE_PATH}/zksign/witness_generator" ]; then
+  echo "ERROR: Missing zksign/witness_generator in restored bundle; ensure the tarball contains host-compatible circuits." >&2
+  exit 1
+fi
+KZG_HOST_PATH="${HOST_BUNDLE_PATH}/${KZG_FILE}"
+if [ ! -f "${KZG_HOST_PATH}" ]; then
+  echo "ERROR: KZG params missing at ${KZG_HOST_PATH}; ensure the tarball contains circuits." >&2
+  exit 1
+fi
+
+if [ "$MODE" = "host" ]; then
+  if [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN}" ]; then
+    echo "==> Using provided host binaries (env override)"
+  else
+    tar_node="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-node"
+    tar_exec="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-executor"
+  if [ ! -x "${tar_node}" ] || [ ! -x "${tar_exec}" ]; then
+    echo "ERROR: Restored tarball missing host executables; provide a host-compatible binaries tarball." >&2
+    exit 1
+  fi
+  if ! host_bin_matches_arch "${tar_node}" || ! host_bin_matches_arch "${tar_exec}"; then
+    echo "ERROR: Restored executables do not match host architecture; provide a host-compatible binaries tarball." >&2
+    exit 1
+  fi
+    echo "==> Using restored host binaries from tarball"
+    NOMOS_NODE_BIN="${tar_node}"
+    NOMOS_EXECUTOR_BIN="${tar_exec}"
+    export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN
   fi
 fi
 

scripts/update-nomos-rev.sh

@@ -1,15 +1,46 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Update nomos-node revision across versions.env and Cargo.toml.
+# Update nomos-node source across versions.env and Cargo.toml.
-# Usage: scripts/update-nomos-rev.sh <new_rev>
+# Usage:
+#   scripts/update-nomos-rev.sh --rev <git_rev>
+#   scripts/update-nomos-rev.sh --path <local_dir>
+#
+# Only one of --rev/--path may be supplied.
 
-if [ "$#" -ne 1 ]; then
+usage() {
-  echo "Usage: $0 <new_nomós_node_rev>" >&2
+  cat <<'EOF'
-  exit 1
+Usage:
+  scripts/update-nomos-rev.sh --rev <git_rev>
+  scripts/update-nomos-rev.sh --path <local_dir>
+
+Notes:
+  --rev   sets NOMOS_NODE_REV and updates Cargo.toml revs
+  --path  sets NOMOS_NODE_PATH (clears NOMOS_NODE_REV) for local checkout
+  Only one may be used at a time.
+EOF
+}
+
+REV=""
+LOCAL_PATH=""
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --rev) REV="${2:-}"; shift 2 ;;
+    --path) LOCAL_PATH="${2:-}"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "Unknown arg: $1" >&2; usage; exit 1 ;;
+  esac
+done
+
+if [ -n "${REV}" ] && [ -n "${LOCAL_PATH}" ]; then
+  echo "Use either --rev or --path, not both" >&2
+  usage; exit 1
+fi
+
+if [ -z "${REV}" ] && [ -z "${LOCAL_PATH}" ]; then
+  usage; exit 1
 fi
 
-NEW_REV="$1"
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 
 if [ ! -f "${ROOT_DIR}/versions.env" ]; then
@@ -17,14 +48,128 @@ if [ ! -f "${ROOT_DIR}/versions.env" ]; then
   exit 1
 fi
 
-echo "Updating nomos-node rev to ${NEW_REV}"
+# Ensure keys exist so sed replacements succeed even if they were absent.
+ensure_env_key() {
+  local key="$1" default_value="$2"
+  if ! grep -Eq "^#?[[:space:]]*${key}=" "${ROOT_DIR}/versions.env"; then
+    echo "${default_value}" >> "${ROOT_DIR}/versions.env"
+  fi
+}
+ensure_env_key "NOMOS_NODE_REV" "# NOMOS_NODE_REV="
+ensure_env_key "NOMOS_NODE_PATH" "# NOMOS_NODE_PATH="
 
-# Update versions.env NOMOS_NODE_REV entry (keep other lines intact).
+if [ -n "${REV}" ]; then
-sed -i.bak -E "s/^NOMOS_NODE_REV=.*/NOMOS_NODE_REV=${NEW_REV}/" "${ROOT_DIR}/versions.env"
+  echo "Updating nomos-node rev to ${REV}"
-rm -f "${ROOT_DIR}/versions.env.bak"
+  # Update versions.env NOMOS_NODE_REV entry, clear NOMOS_NODE_PATH if present.
+  sed -i.bak -E \
+    -e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/NOMOS_NODE_REV=${REV}/" \
+    -e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/# NOMOS_NODE_PATH=/" \
+    "${ROOT_DIR}/versions.env"
+  rm -f "${ROOT_DIR}/versions.env.bak"
 
-# Update all rev fields in Cargo.toml pointing to nomos-node.
+  # Update all rev fields in Cargo.toml pointing to nomos-node and strip any patch block.
-sed -i.bak -E "s/(git = \"https:\/\/github.com\/logos-co\/nomos-node\.git\", rev = \")[^\"]+(\".*)/\1${NEW_REV}\2/" "${ROOT_DIR}/Cargo.toml"
+  python3 - "$ROOT_DIR" "$REV" <<'PY'
-rm -f "${ROOT_DIR}/Cargo.toml.bak"
+import pathlib, re, sys
+root = pathlib.Path(sys.argv[1])
+rev = sys.argv[2]
+cargo_toml = root / "Cargo.toml"
+txt = cargo_toml.read_text()
+txt = txt.replace("\\n", "\n")
+txt = re.sub(
+    r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)',
+    "",
+    txt,
+)
+txt = re.sub(
+    r'(git = "https://github\.com/logos-co/nomos-node\.git", rev = ")[^"]+(")',
+    r"\g<1>" + rev + r"\2",
+    txt,
+)
+cargo_toml.write_text(txt.rstrip() + "\n")
+PY
+else
+  echo "Pointing to local nomos-node at ${LOCAL_PATH}"
+  if [ ! -d "${LOCAL_PATH}" ]; then
+    echo "ERROR: path does not exist: ${LOCAL_PATH}" >&2
+    exit 1
+  fi
+  CURRENT_REV="$(grep -E '^[#[:space:]]*NOMOS_NODE_REV=' "${ROOT_DIR}/versions.env" | head -n1 | sed -E 's/^#?[[:space:]]*NOMOS_NODE_REV=//')"
+  # Update versions.env to favor the local path.
+  sed -i.bak -E \
+    -e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/NOMOS_NODE_PATH=${LOCAL_PATH//\//\\/}/" \
+    -e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/# NOMOS_NODE_REV=${CURRENT_REV}/" \
+    "${ROOT_DIR}/versions.env"
+  rm -f "${ROOT_DIR}/versions.env.bak"
+
+  # Generate a patch block for all nomos-node crates we depend on.
+  PYTHON_BIN="${PYTHON_BIN:-python3}"
+  if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then
+    echo "ERROR: python3 is required to patch Cargo.toml for local paths" >&2
+    exit 1
+  fi
+  "${PYTHON_BIN}" - "$ROOT_DIR" "$LOCAL_PATH" <<'PY'
+import pathlib, subprocess, json, sys, re
+root = pathlib.Path(sys.argv[1])
+node_path = pathlib.Path(sys.argv[2])
+
+targets = [
+    "broadcast-service", "chain-leader", "chain-network", "chain-service",
+    "common-http-client", "cryptarchia-engine", "cryptarchia-sync",
+    "executor-http-client", "groth16", "key-management-system-service",
+    "kzgrs", "kzgrs-backend", "nomos-api", "nomos-blend-message",
+    "nomos-blend-service", "nomos-core", "nomos-da-dispersal",
+    "nomos-da-network-core", "nomos-da-network-service", "nomos-da-sampling",
+    "nomos-da-verifier", "nomos-executor", "nomos-http-api-common",
+    "nomos-ledger", "nomos-libp2p", "nomos-network", "nomos-node",
+    "nomos-sdp", "nomos-time", "nomos-tracing", "nomos-tracing-service",
+    "nomos-utils", "nomos-wallet", "poc", "pol", "subnetworks-assignations",
+    "tests", "tx-service", "wallet", "zksign",
+]
+
+try:
+    meta = subprocess.check_output(
+        ["cargo", "metadata", "--format-version", "1", "--no-deps"],
+        cwd=node_path,
+    )
+except subprocess.CalledProcessError as exc:
+    sys.stderr.write(f"Failed to run cargo metadata in {node_path}: {exc}\\n")
+    sys.exit(1)
+
+data = json.loads(meta)
+paths = {}
+for pkg in data.get("packages", []):
+    paths[pkg["name"]] = str(pathlib.Path(pkg["manifest_path"]).parent)
+
+patch_lines = ['[patch."https://github.com/logos-co/nomos-node"]']
+missing = []
+for name in targets:
+    if name in paths:
+        patch_lines.append(f'{name} = {{ path = "{paths[name]}" }}')
+    else:
+        missing.append(name)
+
+cargo_toml = root / "Cargo.toml"
+txt = cargo_toml.read_text()
+# Normalize any accidental literal escape sequences.
+txt = txt.replace("\\n", "\n")
+# Strip existing patch block for this URL if present (non-greedy).
+txt = re.sub(
+    r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)',
+    "",
+    txt,
+)
+# Ensure a trailing newline and append new patch.
+txt = txt.rstrip() + "\n\n" + "\n".join(patch_lines) + "\n"
+cargo_toml.write_text(txt)
+
+if missing:
+    sys.stderr.write(
+        "Warning: missing crates in local nomos-node checkout: "
+        + ", ".join(missing)
+        + "\\n"
+    )
+PY
+
+fi
 
 echo "Done. Consider updating Cargo.lock if needed (cargo fetch)."

testing-framework/assets/stack/Dockerfile

@@ -81,60 +81,22 @@ RUN mkdir -p /opt/circuits && \
 
 ENV NOMOS_CIRCUITS=/opt/circuits
 
-# Provide runtime binaries. Prefer prebuilt artifacts (when present) for speed; otherwise build from source.
+# Provide runtime binaries. Require prebuilt artifacts staged in testing-framework/assets/stack/bin.
 RUN set -eu; \
     mkdir -p /workspace/artifacts; \
-    TARGET_ARCH="$(uname -m)"; \
+    if [ -f testing-framework/assets/stack/bin/nomos-node ] && [ -f testing-framework/assets/stack/bin/nomos-executor ]; then \
-    expect_arch() { \
+      echo "Using prebuilt nomos binaries from testing-framework/assets/stack/bin (no in-image build)"; \
-      case "$1" in \
-        x86_64) echo "x86-64" ;; \
-        aarch64|arm64) echo "arm64" ;; \
-        *) echo "$1" ;; \
-      esac; \
-    }; \
-    have_prebuilt() { \
-      [ -f testing-framework/assets/stack/bin/nomos-node ] && \
-      [ -f testing-framework/assets/stack/bin/nomos-executor ] && \
-      [ -f testing-framework/assets/stack/bin/nomos-cli ]; \
-    }; \
-    bin_matches_arch() { \
-      BIN_INFO="$(file -b testing-framework/assets/stack/bin/nomos-node 2>/dev/null || true)"; \
-      case "$BIN_INFO" in \
-        *ELF*);; \
-        *) return 1 ;; \
-      esac; \
-      case "$TARGET_ARCH" in \
-        x86_64) PATTERN="x86-64|x86_64" ;; \
-        aarch64|arm64) PATTERN="arm64|aarch64" ;; \
-        *) PATTERN="$(expect_arch "$TARGET_ARCH")" ;; \
-      esac; \
-      [ -n "$BIN_INFO" ] && echo "$BIN_INFO" | grep -Eqi "$PATTERN"; \
-    }; \
-    if have_prebuilt; then \
-      if bin_matches_arch; then \
-        echo "Using prebuilt nomos binaries from testing-framework/assets/stack/bin"; \
-      else \
-        echo "Using prebuilt nomos binaries from testing-framework/assets/stack/bin (arch check skipped)"; \
-      fi; \
       cp testing-framework/assets/stack/bin/nomos-node /workspace/artifacts/nomos-node; \
       cp testing-framework/assets/stack/bin/nomos-executor /workspace/artifacts/nomos-executor; \
       cp testing-framework/assets/stack/bin/nomos-cli /workspace/artifacts/nomos-cli; \
     else \
-      echo "Prebuilt nomos binaries missing or wrong architecture; building from source (rev ${NOMOS_NODE_REV})"; \
+      echo "ERROR: Prebuilt nomos binaries not found in testing-framework/assets/stack/bin; aborting build"; \
-      git clone https://github.com/logos-co/nomos-node.git /tmp/nomos-node && \
+      exit 1; \
-      cd /tmp/nomos-node && \
-      git fetch --depth 1 origin "${NOMOS_NODE_REV}" && \
-      git checkout "${NOMOS_NODE_REV}" && \
-      git reset --hard && git clean -fdx && \
-      # Enable pol-dev-mode via cfg to let POL_PROOF_DEV_MODE short-circuit proofs in tests.
-      RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS=/opt/circuits cargo build --features "testing" \
-        -p nomos-node -p nomos-executor -p nomos-cli; \
-      cp /tmp/nomos-node/target/debug/nomos-node /workspace/artifacts/nomos-node; \
-      cp /tmp/nomos-node/target/debug/nomos-executor /workspace/artifacts/nomos-executor; \
-      cp /tmp/nomos-node/target/debug/nomos-cli /workspace/artifacts/nomos-cli; \
-      rm -rf /tmp/nomos-node/target/debug/incremental; \
     fi
 
+# Strip local path patches so container builds use git sources.
+RUN sed -i '/^\[patch\.\"https:\/\/github.com\/logos-co\/nomos-node\"\]/,/^$/d' /workspace/Cargo.toml
+
 # Build cfgsync binaries from this workspace.
 RUN cargo build --all-features --manifest-path /workspace/testing-framework/tools/cfgsync/Cargo.toml --bins
 RUN cp /workspace/target/debug/cfgsync-server /workspace/artifacts/cfgsync-server && \

testing-framework/assets/stack/cfgsync.yaml

@@ -33,16 +33,16 @@ retry_commitments_limit: 5
 
 # Tracing
 tracing_settings:
-  logger: !Loki
+  # Write node logs to disk for debugging (avoid noisy stdout/trace DNS spam).
-    endpoint: http://loki:3100/
+  logger: !File
-    host_identifier: node
+    directory: /var/log/nomos
-  tracing: !Otlp
+    prefix: node
-    endpoint: http://tempo:4317/
+  # Disable OTLP traces to remove DNS errors; metrics stay enabled below.
-    sample_ratio: 0.5
+  tracing: None
-    service_name: node
   filter: !EnvFilter
     filters:
       nomos: debug
+      cryptarchia: debug
   metrics: !Otlp
     endpoint: http://prometheus:9090/api/v1/otlp/v1/metrics
     host_identifier: node

testing-framework/assets/stack/monitoring/grafana/dashboards/api-dashboard.json

@@ -0,0 +1,460 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "rate(http_requests_total[5m])",
+          "legendFormat": "Requests/sec - {{method}} {{endpoint}}",
+          "refId": "A"
+        }
+      ],
+      "title": "HTTP Request Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "http_requests_total",
+          "legendFormat": "Total Requests - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total HTTP Requests",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "s"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))",
+          "legendFormat": "95th percentile - {{method}} {{endpoint}}",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.50, rate(http_request_duration_seconds_bucket[5m]))",
+          "legendFormat": "50th percentile - {{method}} {{endpoint}}",
+          "refId": "B"
+        }
+      ],
+      "title": "HTTP Request Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(http_requests_failed_total[5m]) / rate(http_requests_total[5m]) * 100",
+          "legendFormat": "Error Rate % - {{method}} {{endpoint}}",
+          "refId": "A"
+        }
+      ],
+      "title": "HTTP Error Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(http_requests_total{status_class=\"2xx\"}[5m])",
+          "legendFormat": "2xx - {{method}} {{endpoint}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(http_requests_total{status_class=\"4xx\"}[5m])",
+          "legendFormat": "4xx - {{method}} {{endpoint}}",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(http_requests_total{status_class=\"5xx\"}[5m])",
+          "legendFormat": "5xx - {{method}} {{endpoint}}",
+          "refId": "C"
+        }
+      ],
+      "title": "HTTP Status Codes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "topk(10, rate(http_requests_total[5m]))",
+          "legendFormat": "{{method}} {{endpoint}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Top API Endpoints",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": ["nomos", "api", "http"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "HTTP API Dashboard",
+  "uid": "api-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

testing-framework/assets/stack/monitoring/grafana/dashboards/api-detailed-dashboard.json

@@ -0,0 +1,395 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(http_requests_total[5m])",
+          "legendFormat": "{{method}} {{endpoint}} - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "HTTP Request Rates by Endpoint",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ms"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "http_request_duration_ms",
+          "legendFormat": "{{method}} {{endpoint}} - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "HTTP Request Duration by Endpoint",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sum by (endpoint) (http_requests_total)",
+          "legendFormat": "{{endpoint}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Requests by Endpoint",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (method) (rate(http_requests_total[5m]))",
+          "legendFormat": "{{method}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Request Rate by HTTP Method",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "topk(10, sum by (endpoint) (rate(http_requests_total[5m])))",
+          "legendFormat": "{{endpoint}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 10 Endpoints by Request Rate",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": ["nomos", "api", "http", "endpoints"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "API Endpoints Detailed Dashboard",
+  "uid": "api-detailed-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

testing-framework/assets/stack/monitoring/grafana/dashboards/blend-dashboard.json

@@ -0,0 +1,681 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "msgps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(blend_messages_sent_total[5m])",
+          "legendFormat": "Sent/sec - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(blend_messages_received_total[5m])",
+          "legendFormat": "Received/sec - {{job}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Message Flow Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "pktps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(blend_mix_packets_processed_total[5m])",
+          "legendFormat": "Mix Packets/sec - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Mix Packet Processing Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "blend_messages_sent_total",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Messages Sent",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 6,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "blend_messages_received_total",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Messages Received",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 12,
+        "y": 8
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "blend_mix_packets_processed_total",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Mix Packets Processed",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 18,
+        "y": 8
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "blend_peers_connected",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Connected Peers",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "blend_peers_connected",
+          "legendFormat": "Connected Peers - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Peer Connectivity Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "msgps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 32
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(blend_outbound_messages_failed_total[5m])",
+          "legendFormat": "Failed/sec - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed Message Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(failed_inbound_messages[5m])",
+          "legendFormat": "Failed Inbound/sec - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(blend_connection_events_total[5m])",
+          "legendFormat": "Connection Events/sec - {{job}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Connection & Error Events",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 40
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "pending_outbound_messages",
+          "legendFormat": "Pending Outbound - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Pending Outbound Messages",
+      "type": "stat"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": ["nomos", "blend", "privacy", "mixnet"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Blend Network Dashboard",
+  "uid": "blend-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

testing-framework/assets/stack/monitoring/grafana/dashboards/da-dashboard.json

@@ -0,0 +1,515 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(da_samples_verified_total[5m])",
+          "legendFormat": "Verified/sec - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(da_samples_failed_total[5m])",
+          "legendFormat": "Failed/sec - {{job}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Sample Verification Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(da_blob_requests_total[5m])",
+          "legendFormat": "Requests/sec - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(da_blob_responses_total[5m])",
+          "legendFormat": "Responses/sec - {{job}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Blob Request/Response Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "da_samples_verified_total",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Samples Verified",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "da_blob_requests_total",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Blob Requests",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 8
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "da_blob_responses_total",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Blob Responses",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "da_inbound_connections",
+          "legendFormat": "Inbound - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "da_outbound_connections",
+          "legendFormat": "Outbound - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "da_subnetwork_connections_total",
+          "legendFormat": "Total - {{job}}",
+          "refId": "C"
+        }
+      ],
+      "title": "DA Network Connections",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(da_samples_verified_total[5m]) / (rate(da_samples_verified_total[5m]) + rate(da_samples_failed_total[5m])) * 100",
+          "legendFormat": "Success Rate % - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Sampling Success Rate",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": ["nomos", "data-availability", "da", "sampling"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Data Availability Dashboard",
+  "uid": "da-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

testing-framework/assets/stack/monitoring/grafana/dashboards/issues-diagnostics-dashboard.json

@@ -0,0 +1,726 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "red",
+                "value": 10
+              }
+            ]
+          },
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "rate(da_samples_failed_total[5m]) * 60",
+          "legendFormat": "DA Samples Failed/min",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(storage_api_operations_failed_total[5m]) * 60",
+          "legendFormat": "Storage Failures/min",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(http_requests_failed_total[5m]) * 60",
+          "legendFormat": "HTTP Failures/min",
+          "refId": "C"
+        },
+        {
+          "expr": "rate(blend_outbound_messages_failed_total[5m]) * 60",
+          "legendFormat": "Blend Msg Failures/min",
+          "refId": "D"
+        },
+        {
+          "expr": "rate(orphan_blocks_fetch_failed_total[5m]) * 60",
+          "legendFormat": "Orphan Block Fetch Failures/min",
+          "refId": "E"
+        },
+        {
+          "expr": "rate(time_broadcast_errors_total[5m]) * 60",
+          "legendFormat": "Time Broadcast Errors/min",
+          "refId": "F"
+        }
+      ],
+      "title": "🚨 Critical Failure Rates (per minute)",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(da_samples_failed_total[5m]) / (rate(da_samples_verified_total[5m]) + rate(da_samples_failed_total[5m])) * 100",
+          "legendFormat": "DA Sample Failure Rate % - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(storage_api_operations_failed_total[5m]) / rate(storage_api_operations_total[5m]) * 100",
+          "legendFormat": "Storage Failure Rate % - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(http_requests_failed_total[5m]) / rate(http_requests_total[5m]) * 100",
+          "legendFormat": "HTTP Failure Rate % - {{job}}",
+          "refId": "C"
+        }
+      ],
+      "title": "📊 Failure Rates (Percentage)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(kms_register_failures_total[5m])",
+          "legendFormat": "KMS Registration Failures - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(kms_sign_failures_total[5m])",
+          "legendFormat": "KMS Signing Failures - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(kms_execute_failures_total[5m])",
+          "legendFormat": "KMS Execute Failures - {{job}}",
+          "refId": "C"
+        }
+      ],
+      "title": "🔐 KMS Failures",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(sdp_declaration_tx_failures_total[5m])",
+          "legendFormat": "SDP Declaration TX Failures - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(sdp_declaration_mempool_failures_total[5m])",
+          "legendFormat": "SDP Declaration Mempool Failures - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(sdp_activity_tx_failures_total[5m])",
+          "legendFormat": "SDP Activity TX Failures - {{job}}",
+          "refId": "C"
+        },
+        {
+          "expr": "rate(sdp_withdrawal_validation_failures_total[5m])",
+          "legendFormat": "SDP Withdrawal Validation Failures - {{job}}",
+          "refId": "D"
+        }
+      ],
+      "title": "📋 SDP (Service Provider) Failures",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(time_broadcast_errors_total[5m])",
+          "legendFormat": "Time Broadcast Errors - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(time_lagging_errors_total[5m])",
+          "legendFormat": "Time Lagging Errors - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(time_subscription_errors_total[5m])",
+          "legendFormat": "Time Subscription Errors - {{job}}",
+          "refId": "C"
+        },
+        {
+          "expr": "rate(time_current_slot_errors_total[5m])",
+          "legendFormat": "Time Current Slot Errors - {{job}}",
+          "refId": "D"
+        }
+      ],
+      "title": "⏰ Timing/Synchronization Issues",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(network_pubsub_subscription_errors_total[5m])",
+          "legendFormat": "PubSub Subscription Errors - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(network_chainsync_subscription_errors_total[5m])",
+          "legendFormat": "ChainSync Subscription Errors - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(sdp_subscription_errors_total[5m])",
+          "legendFormat": "SDP Subscription Errors - {{job}}",
+          "refId": "C"
+        }
+      ],
+      "title": "🌐 Network/Subscription Errors",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.1
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          },
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "id": 7,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_tip_height - consensus_finalized_height",
+          "legendFormat": "Unfinalized Blocks - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "orphan_blocks_pending",
+          "legendFormat": "Pending Orphan Blocks - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "mempool_transactions_pending",
+          "legendFormat": "Pending Mempool Txs - {{job}}",
+          "refId": "C"
+        }
+      ],
+      "title": "⚠️ Potential Bottlenecks",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 32
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "topk(10, rate(storage_api_operations_failed_total[5m]))",
+          "legendFormat": "{{operation}} ({{api}}) - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "🗄️ Top Storage Operation Failures",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": ["nomos", "diagnostics", "errors", "issues", "alerting"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "🚨 Issues & Diagnostics Dashboard",
+  "uid": "issues-diagnostics-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

testing-framework/assets/stack/monitoring/grafana/dashboards/mempool-dashboard.json

@@ -0,0 +1,555 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "mempool_transactions_pending",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Pending Transactions",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "mempool_transactions_added_total",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Added",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "mempool_transactions_removed_total",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Total Removed",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "mempool_transactions_pending",
+          "legendFormat": "Pending - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Pending Transactions Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "tps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(mempool_transactions_added_total[5m])",
+          "legendFormat": "Added/sec - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(mempool_transactions_removed_total[5m])",
+          "legendFormat": "Removed/sec - {{job}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Transaction Processing Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "mempool_transactions_added_total",
+          "legendFormat": "Added - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "mempool_transactions_removed_total",
+          "legendFormat": "Removed - {{job}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Cumulative Transaction Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(mempool_transactions_removed_total[5m]) / rate(mempool_transactions_added_total[5m]) * 100",
+          "legendFormat": "Processing Efficiency % - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Mempool Processing Efficiency",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "id": 8,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "tx_mempool_pending_items",
+          "legendFormat": "TX Service Pending - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "TX Service Mempool Metrics",
+      "type": "stat"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": ["nomos", "mempool", "transactions"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Mempool Dashboard",
+  "uid": "mempool-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

testing-framework/assets/stack/monitoring/grafana/dashboards/services-dashboard.json

@@ -0,0 +1,676 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "kms_register_success_total",
+          "legendFormat": "Registered Keys - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "KMS Keys Registered",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "kms_sign_success_total",
+          "legendFormat": "Signatures - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "KMS Signatures Total",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sdp_declarations_total",
+          "legendFormat": "Declarations - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "SDP Declarations Total",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "sdp_activity_posts_total",
+          "legendFormat": "Activities - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "SDP Activities Total",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(kms_sign_requests_total[5m])",
+          "legendFormat": "Sign Requests/sec - {{strategy}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(kms_register_requests_total[5m])",
+          "legendFormat": "Register Requests/sec",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(kms_public_key_requests_total[5m])",
+          "legendFormat": "PubKey Requests/sec",
+          "refId": "C"
+        }
+      ],
+      "title": "KMS Request Rates",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "percent"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(kms_sign_failures_total[5m]) / rate(kms_sign_requests_total[5m]) * 100",
+          "legendFormat": "KMS Sign Error % - {{strategy}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(kms_register_failures_total[5m]) / rate(kms_register_requests_total[5m]) * 100",
+          "legendFormat": "KMS Register Error %",
+          "refId": "B"
+        }
+      ],
+      "title": "KMS Error Rates",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(sdp_declarations_total[5m])",
+          "legendFormat": "Declarations/sec",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(sdp_activity_posts_total[5m])",
+          "legendFormat": "Activities/sec",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(sdp_withdrawals_total[5m])",
+          "legendFormat": "Withdrawals/sec",
+          "refId": "C"
+        }
+      ],
+      "title": "SDP Operation Rates",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(sdp_declaration_success_total[5m])",
+          "legendFormat": "Declaration Success/sec",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(sdp_declaration_tx_failures_total[5m])",
+          "legendFormat": "Declaration TX Failures/sec",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(sdp_activity_success_total[5m])",
+          "legendFormat": "Activity Success/sec",
+          "refId": "C"
+        },
+        {
+          "expr": "rate(sdp_withdrawal_success_total[5m])",
+          "legendFormat": "Withdrawal Success/sec",
+          "refId": "D"
+        }
+      ],
+      "title": "SDP Success/Failure Rates",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 24
+      },
+      "id": 9,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(time_subscription_requests_total[5m])",
+          "legendFormat": "Time Subscriptions/sec",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(network_pubsub_subscriptions_total[5m])",
+          "legendFormat": "PubSub Subscriptions/sec",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(network_chainsync_subscriptions_total[5m])",
+          "legendFormat": "ChainSync Subscriptions/sec",
+          "refId": "C"
+        },
+        {
+          "expr": "rate(sdp_subscriptions_total[5m])",
+          "legendFormat": "SDP Subscriptions/sec",
+          "refId": "D"
+        }
+      ],
+      "title": "Service Subscription Activity",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": ["nomos", "services", "kms", "sdp", "time"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Services Dashboard",
+  "uid": "services-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

testing-framework/assets/stack/monitoring/grafana/dashboards/storage-dashboard.json

@@ -0,0 +1,445 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(storage_api_operations_total{api=\"da\"}[5m])",
+          "legendFormat": "DA Operations/sec - {{operation}} - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "DA Storage Operations Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(storage_api_operations_total{api=\"chain\"}[5m])",
+          "legendFormat": "Chain Operations/sec - {{operation}} - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Chain Storage Operations Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 16
+      },
+      "id": 3,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(storage_api_operations_failed_total{api=\"da\"}[5m])",
+          "legendFormat": "DA Failures/sec - {{operation}} - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "DA Storage Failures Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "ops"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(storage_api_operations_failed_total{api=\"chain\"}[5m])",
+          "legendFormat": "Chain Failures/sec - {{operation}} - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Chain Storage Failures Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "storage_api_operations_total{api=\"da\"}",
+          "legendFormat": "DA Total - {{operation}} - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "DA Storage Operations Total",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 8
+      },
+      "id": 6,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "storage_api_operations_total{api=\"chain\"}",
+          "legendFormat": "Chain Total - {{operation}} - {{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Chain Storage Operations Total",
+      "type": "stat"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": ["nomos", "storage", "rocksdb"],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Storage Dashboard",
+  "uid": "storage-dashboard",
+  "version": 1,
+  "weekStart": ""
+}

testing-framework/assets/stack/monitoring/grafana/dashboards/testnet_metrics.json

@@ -105,7 +105,7 @@
         {
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "da_mempool_pending_items",
+          "expr": "mempool_transactions_pending",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "legendFormat": "__auto",
@@ -205,7 +205,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "consensus_processed_blocks",
+          "expr": "rate(consensus_blocks_imported_total[5m])",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "legendFormat": "__auto",
@@ -214,7 +214,976 @@
           "useBackend": false
         }
       ],
-      "title": "Consensus: Processed Blocks",
+      "title": "Consensus: Block Import Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 0,
+        "y": 16
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_tip_height",
+          "refId": "A"
+        }
+      ],
+      "title": "Consensus Tip Height",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 4,
+        "y": 16
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_finalized_height",
+          "refId": "A"
+        }
+      ],
+      "title": "Finalized Height",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 8,
+        "y": 16
+      },
+      "id": 5,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_current_epoch",
+          "refId": "A"
+        }
+      ],
+      "title": "Current Epoch",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 16
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(da_samples_verified_total[5m])",
+          "legendFormat": "Verified/sec",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(da_samples_failed_total[5m])",
+          "legendFormat": "Failed/sec",
+          "refId": "B"
+        }
+      ],
+      "title": "DA Sample Verification Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "consensus_peers_connected",
+          "legendFormat": "Connected Peers",
+          "refId": "A"
+        },
+        {
+          "expr": "consensus_connections_total",
+          "legendFormat": "Total Connections",
+          "refId": "B"
+        }
+      ],
+      "title": "Consensus Network",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 24
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "blend_peers_connected",
+          "legendFormat": "Connected Peers",
+          "refId": "A"
+        }
+      ],
+      "title": "Blend Network",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 0,
+        "y": 32
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_tip_height",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Consensus Tip Height",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 4,
+        "y": 32
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_finalized_height",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Consensus Finalized Height",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 8,
+        "y": 32
+      },
+      "id": 11,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_current_epoch",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Current Epoch",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 12,
+        "y": 32
+      },
+      "id": 12,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_current_slot",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Current Slot",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "vis": false
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 16,
+        "y": 32
+      },
+      "id": 13,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "values": false,
+          "calcs": ["lastNotNull"],
+          "fields": ""
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "expr": "consensus_branches_count",
+          "legendFormat": "{{job}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Consensus Branches",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 40
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "consensus_peers_connected",
+          "legendFormat": "Connected Peers - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "consensus_connections_total",
+          "legendFormat": "Total Connections - {{job}}",
+          "refId": "B"
+        }
+      ],
+      "title": "Consensus Network Connections",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "short"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 40
+      },
+      "id": 15,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "mempool_transactions_pending",
+          "legendFormat": "Pending - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(mempool_transactions_added_total[5m])",
+          "legendFormat": "Added/sec - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(mempool_transactions_removed_total[5m])",
+          "legendFormat": "Removed/sec - {{job}}",
+          "refId": "C"
+        }
+      ],
+      "title": "Mempool Transactions",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 48
+      },
+      "id": 16,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(da_blob_requests_total[5m])",
+          "legendFormat": "Requests/sec - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(da_blob_responses_total[5m])",
+          "legendFormat": "Responses/sec - {{job}}",
+          "refId": "B"
+        }
+      ],
+      "title": "DA Blob Request/Response Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "unit": "msgps"
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 48
+      },
+      "id": 17,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "rate(blend_messages_sent_total[5m])",
+          "legendFormat": "Sent/sec - {{job}}",
+          "refId": "A"
+        },
+        {
+          "expr": "rate(blend_messages_received_total[5m])",
+          "legendFormat": "Received/sec - {{job}}",
+          "refId": "B"
+        },
+        {
+          "expr": "rate(blend_mix_packets_processed_total[5m])",
+          "legendFormat": "Mix Packets/sec - {{job}}",
+          "refId": "C"
+        }
+      ],
+      "title": "Blend Message Flow Rate",
       "type": "timeseries"
     }
   ],

testing-framework/assets/stack/monitoring/grafana/datasources.yaml

@@ -9,29 +9,3 @@ datasources:
     is_default: true
     version: 1
     editable: true
-
-  - name: Tempo
-    type: tempo
-    access: proxy
-    org_id: 1
-    url: http://tempo:3200
-    is_default: false
-    version: 1
-    editable: true
-    uid: tempods
-
-  - name: Loki
-    type: loki
-    access: proxy
-    org_id: 1
-    url: http://loki:3100
-    is_default: false
-    version: 1
-    editable: true
-    jsonData:
-      derivedFields:
-        - name: trace_id
-          matcherRegex: "\"trace_id\":\"(\\w+)\""
-          url: "$${__value.raw}"
-          datasourceUid: tempods
-

testing-framework/assets/stack/scripts/build_test_image.sh

@@ -34,6 +34,36 @@ echo "Image tag: ${IMAGE_TAG}"
 echo "Circuits override: ${CIRCUITS_OVERRIDE:-<none>}"
 echo "Circuits version (fallback download): ${VERSION}"
 echo "Circuits platform: ${CIRCUITS_PLATFORM}"
+echo "Bundle tar (if used): ${NOMOS_BINARIES_TAR:-<default>.tmp/nomos-binaries-linux-${VERSION}.tar.gz}"
+
+# If prebuilt binaries are missing, restore them from a bundle tarball instead of
+# rebuilding nomos inside the image.
+BIN_DST="${ROOT_DIR}/testing-framework/assets/stack/bin"
+DEFAULT_LINUX_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz"
+TAR_PATH="${NOMOS_BINARIES_TAR:-${DEFAULT_LINUX_TAR}}"
+
+if [ ! -x "${BIN_DST}/nomos-node" ] || [ ! -x "${BIN_DST}/nomos-executor" ]; then
+  if [ -f "${TAR_PATH}" ]; then
+    echo "Restoring binaries/circuits from ${TAR_PATH}"
+    tmp_extract="$(mktemp -d)"
+    tar -xzf "${TAR_PATH}" -C "${tmp_extract}"
+    if [ -f "${tmp_extract}/artifacts/nomos-node" ] && [ -f "${tmp_extract}/artifacts/nomos-executor" ]; then
+      mkdir -p "${BIN_DST}"
+      cp "${tmp_extract}/artifacts/nomos-node" "${tmp_extract}/artifacts/nomos-executor" "${tmp_extract}/artifacts/nomos-cli" "${BIN_DST}/"
+    else
+      echo "ERROR: Bundle ${TAR_PATH} missing binaries under artifacts/" >&2
+      exit 1
+    fi
+    if [ -d "${tmp_extract}/artifacts/circuits" ]; then
+      mkdir -p "${KZG_DIR_REL}"
+      rsync -a --delete "${tmp_extract}/artifacts/circuits/" "${KZG_DIR_REL}/"
+    fi
+    rm -rf "${tmp_extract}"
+  else
+    echo "ERROR: Prebuilt binaries missing and bundle tar not found at ${TAR_PATH}" >&2
+    exit 1
+  fi
+fi
 
 build_args=(
   -f "${DOCKERFILE_PATH}"

testing-framework/configs/src/nodes/blend.rs

@@ -1,7 +1,7 @@
 use std::{num::NonZeroU64, path::PathBuf, time::Duration};
 
 use blend_serde::Config as BlendUserConfig;
-use key_management_system_service::keys::{Key, ZkKey};
+use key_management_system_service::keys::Key;
 use nomos_blend_service::{
     core::settings::{CoverTrafficSettings, MessageDelayerSettings, SchedulerSettings, ZkSettings},
     settings::TimingSettings,
@@ -27,18 +27,16 @@ pub(crate) fn build_blend_service_config(
     BlendDeploymentSettings,
     NetworkDeploymentSettings,
 ) {
-    let zk_key_id =
+    let zk_key_id = key_id_for_preload_backend(&Key::from(config.secret_zk_key.clone()));
-        key_id_for_preload_backend(&Key::from(ZkKey::new(config.secret_zk_key.clone())));
 
     let backend_core = &config.backend_core;
     let backend_edge = &config.backend_edge;
 
     let user = BlendUserConfig {
-        common: blend_serde::common::Config {
+        non_ephemeral_signing_key: config.private_key.clone(),
-            non_ephemeral_signing_key: config.private_key.clone(),
+        // Persist recovery data under the tempdir so components expecting it
-            // Disable on-disk recovery in tests to avoid serde issues on replays.
+        // can start cleanly.
-            recovery_path_prefix: PathBuf::new(),
+        recovery_path_prefix: PathBuf::from("./recovery/blend"),
-        },
         core: blend_serde::core::Config {
             backend: blend_serde::core::BackendConfig {
                 listening_address: backend_core.listening_address.clone(),

testing-framework/configs/src/nodes/common.rs

@@ -21,8 +21,7 @@ use nomos_node::{
                 SdpConfig as DeploymentSdpConfig, Settings as CryptarchiaDeploymentSettings,
             },
             serde::{
-                Config as CryptarchiaConfig, LeaderConfig as CryptarchiaLeaderConfig,
+                Config as CryptarchiaConfig, NetworkConfig as CryptarchiaNetworkConfig,
-                NetworkConfig as CryptarchiaNetworkConfig,
                 ServiceConfig as CryptarchiaServiceConfig,
             },
         },
@@ -96,11 +95,9 @@ pub(crate) fn cryptarchia_config(config: &GeneralConfig) -> CryptarchiaConfig {
                 },
             },
         },
-        leader: CryptarchiaLeaderConfig {
+        leader: ChainLeaderConfig {
-            leader: ChainLeaderConfig {
+            pk: config.consensus_config.leader_config.pk,
-                pk: config.consensus_config.leader_config.pk,
+            sk: config.consensus_config.leader_config.sk.clone(),
-                sk: config.consensus_config.leader_config.sk.clone(),
-            },
         },
     }
 }

testing-framework/configs/src/topology/configs/blend.rs

@@ -1,23 +1,21 @@
 use core::time::Duration;
 use std::{num::NonZeroU64, str::FromStr as _};
 
-use ed25519_dalek::SigningKey;
+use key_management_system_service::keys::{Ed25519Key, UnsecuredEd25519Key, ZkKey};
-use key_management_system_service::keys::UnsecuredEd25519Key;
 use nomos_blend_service::{
     core::backends::libp2p::Libp2pBlendBackendSettings as Libp2pCoreBlendBackendSettings,
     edge::backends::libp2p::Libp2pBlendBackendSettings as Libp2pEdgeBlendBackendSettings,
 };
 use nomos_libp2p::{Multiaddr, protocol_name::StreamProtocol};
 use num_bigint::BigUint;
-use zksign::SecretKey;
 
 #[derive(Clone)]
 pub struct GeneralBlendConfig {
     pub backend_core: Libp2pCoreBlendBackendSettings,
     pub backend_edge: Libp2pEdgeBlendBackendSettings,
     pub private_key: UnsecuredEd25519Key,
-    pub secret_zk_key: SecretKey,
+    pub secret_zk_key: ZkKey,
-    pub signer: SigningKey,
+    pub signer: Ed25519Key,
 }
 
 /// Builds blend configs for each node.
@@ -32,15 +30,13 @@ pub fn create_blend_configs(ids: &[[u8; 32]], ports: &[u16]) -> Vec<GeneralBlend
     ids.iter()
         .zip(ports)
         .map(|(id, port)| {
-            let signer = SigningKey::from_bytes(id);
+            let signer = Ed25519Key::from_bytes(id);
-
+            let private_key = UnsecuredEd25519Key::from_bytes(id);
-            let private_key = UnsecuredEd25519Key::from(signer.clone());
             // We need unique ZK secret keys, so we just derive them deterministically from
             // the generated Ed25519 public keys, which are guaranteed to be unique because
             // they are in turned derived from node ID.
-            let secret_zk_key = SecretKey::from(BigUint::from_bytes_le(
+            let secret_zk_key =
-                private_key.as_ref().verifying_key().as_bytes(),
+                ZkKey::from(BigUint::from_bytes_le(private_key.public_key().as_bytes()));
-            ));
             GeneralBlendConfig {
                 backend_core: Libp2pCoreBlendBackendSettings {
                     listening_address: Multiaddr::from_str(&format!(

testing-framework/configs/src/topology/configs/consensus.rs

@@ -5,8 +5,10 @@ use std::{
 
 use chain_leader::LeaderConfig;
 use cryptarchia_engine::EpochConfig;
-use ed25519_dalek::ed25519::signature::SignerMut as _;
 use groth16::CompressedGroth16Proof;
+use key_management_system_service::keys::{
+    Ed25519Key, UnsecuredZkKey, ZkKey, ZkPublicKey, ZkSignature,
+};
 use nomos_core::{
     mantle::{
         MantleTx, Note, OpProof, Utxo,
@@ -22,7 +24,6 @@ use nomos_core::{
 use nomos_node::{SignedMantleTx, Transaction as _};
 use nomos_utils::math::NonNegativeF64;
 use num_bigint::BigUint;
-use zksign::{PublicKey, SecretKey};
 
 use super::wallet::{WalletAccount, WalletConfig};
 
@@ -52,8 +53,8 @@ impl ConsensusParams {
 #[derive(Clone)]
 pub struct ProviderInfo {
     pub service_type: ServiceType,
-    pub provider_sk: ed25519_dalek::SigningKey,
+    pub provider_sk: Ed25519Key,
-    pub zk_sk: SecretKey,
+    pub zk_sk: ZkKey,
     pub locator: Locator,
     pub note: ServiceNote,
 }
@@ -61,11 +62,11 @@ pub struct ProviderInfo {
 impl ProviderInfo {
     #[must_use]
     pub fn provider_id(&self) -> ProviderId {
-        ProviderId(self.provider_sk.verifying_key())
+        ProviderId(self.provider_sk.public_key())
     }
 
     #[must_use]
-    pub fn zk_id(&self) -> PublicKey {
+    pub fn zk_id(&self) -> ZkPublicKey {
         self.zk_sk.to_public_key()
     }
 }
@@ -85,8 +86,8 @@ pub struct GeneralConsensusConfig {
 
 #[derive(Clone)]
 pub struct ServiceNote {
-    pub pk: PublicKey,
+    pub pk: ZkPublicKey,
-    pub sk: SecretKey,
+    pub sk: ZkKey,
     pub note: Note,
     pub output_index: usize,
 }
@@ -114,7 +115,7 @@ fn create_genesis_tx(utxos: &[Utxo]) -> GenesisTx {
     let signed_mantle_tx = SignedMantleTx {
         mantle_tx,
         ops_proofs: vec![OpProof::NoProof],
-        ledger_tx_proof: zksign::Signature::new(CompressedGroth16Proof::from_bytes(&[0u8; 128])),
+        ledger_tx_proof: ZkSignature::new(CompressedGroth16Proof::from_bytes(&[0u8; 128])),
     };
 
     // Wrap in GenesisTx
@@ -206,7 +207,7 @@ pub fn create_consensus_configs(
 
 fn create_utxos_for_leader_and_services(
     ids: &[[u8; 32]],
-    leader_keys: &mut Vec<(PublicKey, SecretKey)>,
+    leader_keys: &mut Vec<(ZkPublicKey, UnsecuredZkKey)>,
     blend_notes: &mut Vec<ServiceNote>,
     da_notes: &mut Vec<ServiceNote>,
 ) -> Vec<Utxo> {
@@ -229,7 +230,7 @@ fn create_utxos_for_leader_and_services(
     // Create notes for leader, Blend and DA declarations.
     for &id in ids {
         let sk_leader_data = derive_key_material(b"ld", &id);
-        let sk_leader = SecretKey::from(BigUint::from_bytes_le(&sk_leader_data));
+        let sk_leader = UnsecuredZkKey::from(BigUint::from_bytes_le(&sk_leader_data));
         let pk_leader = sk_leader.to_public_key();
         leader_keys.push((pk_leader, sk_leader));
         utxos.push(Utxo {
@@ -240,7 +241,7 @@ fn create_utxos_for_leader_and_services(
         output_index += 1;
 
         let sk_da_data = derive_key_material(b"da", &id);
-        let sk_da = SecretKey::from(BigUint::from_bytes_le(&sk_da_data));
+        let sk_da = ZkKey::from(BigUint::from_bytes_le(&sk_da_data));
         let pk_da = sk_da.to_public_key();
         let note_da = Note::new(1, pk_da);
         da_notes.push(ServiceNote {
@@ -257,7 +258,7 @@ fn create_utxos_for_leader_and_services(
         output_index += 1;
 
         let sk_blend_data = derive_key_material(b"bn", &id);
-        let sk_blend = SecretKey::from(BigUint::from_bytes_le(&sk_blend_data));
+        let sk_blend = ZkKey::from(BigUint::from_bytes_le(&sk_blend_data));
         let pk_blend = sk_blend.to_public_key();
         let note_blend = Note::new(1, pk_blend);
         blend_notes.push(ServiceNote {
@@ -331,13 +332,13 @@ pub fn create_genesis_tx_with_declarations(
     let mantle_tx_hash = mantle_tx.hash();
     let mut ops_proofs = vec![OpProof::NoProof];
 
-    for mut provider in providers {
+    for provider in providers {
         let zk_sig =
-            SecretKey::multi_sign(&[provider.note.sk, provider.zk_sk], mantle_tx_hash.as_ref())
+            ZkKey::multi_sign(&[provider.note.sk, provider.zk_sk], mantle_tx_hash.as_ref())
                 .unwrap();
         let ed25519_sig = provider
             .provider_sk
-            .sign(mantle_tx_hash.as_signing_bytes().as_ref());
+            .sign_payload(mantle_tx_hash.as_signing_bytes().as_ref());
 
         ops_proofs.push(OpProof::ZkAndEd25519Sigs {
             zk_sig,
@@ -348,7 +349,7 @@ pub fn create_genesis_tx_with_declarations(
     let signed_mantle_tx = SignedMantleTx {
         mantle_tx,
         ops_proofs,
-        ledger_tx_proof: zksign::Signature::new(CompressedGroth16Proof::from_bytes(&[0u8; 128])),
+        ledger_tx_proof: ZkSignature::new(CompressedGroth16Proof::from_bytes(&[0u8; 128])),
     };
 
     GenesisTx::from_tx(signed_mantle_tx).expect("Invalid genesis transaction")

testing-framework/configs/src/topology/configs/da.rs

@@ -8,7 +8,7 @@ use std::{
     time::Duration,
 };
 
-use ed25519_dalek::SigningKey;
+use key_management_system_service::keys::{Ed25519Key, ZkKey};
 use nomos_core::sdp::SessionNumber;
 use nomos_da_network_core::swarm::{
     DAConnectionMonitorSettings, DAConnectionPolicySettings, ReplicationConfig,
@@ -19,7 +19,6 @@ use num_bigint::BigUint;
 use rand::random;
 use subnetworks_assignations::{MembershipCreator as _, MembershipHandler as _};
 use tracing::warn;
-use zksign::SecretKey;
 
 use crate::secret_key_to_peer_id;
 
@@ -133,7 +132,7 @@ impl Default for DaParams {
 #[derive(Debug, Clone)]
 pub struct GeneralDaConfig {
     pub node_key: ed25519::SecretKey,
-    pub signer: SigningKey,
+    pub signer: Ed25519Key,
     pub peer_id: PeerId,
     pub membership: NomosDaMembership,
     pub listening_address: Multiaddr,
@@ -153,7 +152,7 @@ pub struct GeneralDaConfig {
     pub subnets_refresh_interval: Duration,
     pub retry_shares_limit: usize,
     pub retry_commitments_limit: usize,
-    pub secret_zk_key: SecretKey,
+    pub secret_zk_key: ZkKey,
 }
 
 #[must_use]
@@ -228,14 +227,13 @@ pub fn create_da_configs(
             let verifier_sk = blst::min_sig::SecretKey::key_gen(id, &[]).unwrap();
             let verifier_sk_bytes = verifier_sk.to_bytes();
             let peer_id = peer_ids[i];
-            let signer = SigningKey::from_bytes(id);
+            let signer = Ed25519Key::from_bytes(id);
             let subnetwork_ids = membership.membership(&peer_id);
 
             // We need unique ZK secret keys, so we just derive them deterministically from
             // the generated Ed25519 public keys, which are guaranteed to be unique because
             // they are in turned derived from node ID.
-            let secret_zk_key =
+            let secret_zk_key = ZkKey::from(BigUint::from_bytes_le(signer.public_key().as_bytes()));
-                SecretKey::from(BigUint::from_bytes_le(signer.verifying_key().as_bytes()));
 
             GeneralDaConfig {
                 node_key,

testing-framework/configs/src/topology/configs/mod.rs

@@ -11,10 +11,7 @@ pub mod wallet;
 use blend::GeneralBlendConfig;
 use consensus::{GeneralConsensusConfig, ProviderInfo, create_genesis_tx_with_declarations};
 use da::GeneralDaConfig;
-use key_management_system_service::{
+use key_management_system_service::{backend::preload::PreloadKMSBackendSettings, keys::Key};
-    backend::preload::PreloadKMSBackendSettings,
-    keys::{Ed25519Key, Key, ZkKey},
-};
 use network::GeneralNetworkConfig;
 use nomos_core::{
     mantle::GenesisTx as _,
@@ -126,8 +123,8 @@ pub fn create_general_configs_with_blend_core_subset(
     let kms_configs: Vec<_> = blend_configs
         .iter()
         .map(|blend_conf| {
-            let ed_key = Ed25519Key::new(blend_conf.signer.clone());
+            let ed_key = blend_conf.signer.clone();
-            let zk_key = ZkKey::new(blend_conf.secret_zk_key.clone());
+            let zk_key = blend_conf.secret_zk_key.clone();
             PreloadKMSBackendSettings {
                 keys: [
                     (

testing-framework/configs/src/topology/configs/network.rs

@@ -82,7 +82,7 @@ pub fn create_network_configs(
         .map(|(swarm_config, initial_peers)| GeneralNetworkConfig {
             backend: BackendSettings {
                 initial_peers,
-                inner: swarm_config.to_owned(),
+                swarm: swarm_config.to_owned(),
             },
         })
         .collect()

testing-framework/configs/src/topology/configs/wallet.rs

@@ -1,7 +1,7 @@
 use std::num::NonZeroUsize;
 
+use key_management_system_service::keys::{ZkKey, ZkPublicKey};
 use num_bigint::BigUint;
-use zksign::{PublicKey, SecretKey};
 
 /// Collection of wallet accounts that should be funded at genesis.
 #[derive(Clone, Default, Debug, serde::Serialize, serde::Deserialize)]
@@ -47,13 +47,13 @@ impl WalletConfig {
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 pub struct WalletAccount {
     pub label: String,
-    pub secret_key: SecretKey,
+    pub secret_key: ZkKey,
     pub value: u64,
 }
 
 impl WalletAccount {
     #[must_use]
-    pub fn new(label: impl Into<String>, secret_key: SecretKey, value: u64) -> Self {
+    pub fn new(label: impl Into<String>, secret_key: ZkKey, value: u64) -> Self {
         assert!(value > 0, "wallet account value must be positive");
         Self {
             label: label.into(),
@@ -68,12 +68,12 @@ impl WalletAccount {
         seed[..2].copy_from_slice(b"wl");
         seed[2..10].copy_from_slice(&index.to_le_bytes());
 
-        let secret_key = SecretKey::from(BigUint::from_bytes_le(&seed));
+        let secret_key = ZkKey::from(BigUint::from_bytes_le(&seed));
         Self::new(format!("wallet-user-{index}"), secret_key, value)
     }
 
     #[must_use]
-    pub fn public_key(&self) -> PublicKey {
+    pub fn public_key(&self) -> ZkPublicKey {
         self.secret_key.to_public_key()
     }
 }

testing-framework/core/src/nodes/api_client.rs

@@ -16,6 +16,7 @@ use nomos_node::{HeaderId, api::testing::handlers::HistoricSamplingRequest};
 use reqwest::{Client, RequestBuilder, Response, Url};
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::Value;
+use tracing::error;
 
 pub const DA_GET_TESTING_ENDPOINT_ERROR: &str = "Failed to connect to testing endpoint. The binary was likely built without the 'testing' \
      feature. Try: cargo build --workspace --all-features";
@@ -276,7 +277,17 @@ impl ApiClient {
 
     /// Submit a mantle transaction through the base API.
     pub async fn submit_transaction(&self, tx: &SignedMantleTx) -> reqwest::Result<()> {
-        self.post_json_unit(MEMPOOL_ADD_TX, tx).await
+        let res = self.post_json_response(MEMPOOL_ADD_TX, tx).await?;
+        if let Err(status_err) = res.error_for_status_ref() {
+            let status = res.status();
+            let body = res
+                .text()
+                .await
+                .unwrap_or_else(|_| "<unreadable body>".to_string());
+            error!(%status, %body, "submit_transaction request failed");
+            return Err(status_err);
+        }
+        Ok(())
     }
 
     /// Execute a custom request built by the caller.

testing-framework/core/src/nodes/common/config/injection.rs

@@ -1,5 +1,34 @@
 use serde_yaml::{Mapping, Number as YamlNumber, Value};
 
+/// Convert any ed25519_sig entries from a YAML sequence of bytes into the
+/// hex-encoded string format expected by nomos-node config deserialization.
+pub fn normalize_ed25519_sigs(value: &mut Value) {
+    match value {
+        Value::Mapping(map) => {
+            for (k, v) in map.iter_mut() {
+                if let Value::String(key) = k {
+                    if key == "ed25519_sig" {
+                        if let Value::Sequence(seq) = v {
+                            let bytes: Option<Vec<u8>> = seq
+                                .iter()
+                                .map(|val| val.as_i64().and_then(|n| u8::try_from(n).ok()))
+                                .collect();
+                            if let Some(bytes) = bytes {
+                                *v = Value::String(hex::encode(bytes));
+                                continue;
+                            }
+                        }
+                    }
+                }
+                normalize_ed25519_sigs(v);
+            }
+        }
+        Value::Sequence(seq) => seq.iter_mut().for_each(normalize_ed25519_sigs),
+        Value::Tagged(tagged) => normalize_ed25519_sigs(&mut tagged.value),
+        _ => {}
+    }
+}
+
 /// Inject cryptarchia/IBD defaults into a YAML config in-place.
 pub fn inject_ibd_into_cryptarchia(yaml_value: &mut Value) {
     let Some(cryptarchia) = cryptarchia_section(yaml_value) else {
@@ -8,6 +37,7 @@ pub fn inject_ibd_into_cryptarchia(yaml_value: &mut Value) {
     ensure_network_adapter(cryptarchia);
     ensure_sync_defaults(cryptarchia);
     ensure_ibd_bootstrap(cryptarchia);
+    normalize_ed25519_sigs(yaml_value);
 }
 
 fn cryptarchia_section(yaml_value: &mut Value) -> Option<&mut Mapping> {

testing-framework/core/src/nodes/common/lifecycle/spawn.rs

@@ -7,6 +7,8 @@ use serde::Serialize;
 use serde_yaml::Value;
 use tracing::debug;
 
+use crate::nodes::common::config::injection::normalize_ed25519_sigs;
+
 /// Configure tracing logger to write into `NOMOS_LOG_DIR` if set, else into the
 /// provided base dir.
 pub fn configure_logging<F>(base_dir: &Path, prefix: &str, set_logger: F)
@@ -40,6 +42,7 @@ where
     let mut yaml_value =
         serde_yaml::to_value(config).map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
     inject(&mut yaml_value);
+    normalize_ed25519_sigs(&mut yaml_value);
     let file = File::create(path)?;
     serde_yaml::to_writer(file, &yaml_value)
         .map_err(|err| io::Error::new(io::ErrorKind::Other, err))

testing-framework/core/src/nodes/common/node.rs

@@ -16,7 +16,7 @@ use super::lifecycle::monitor::is_running;
 use crate::nodes::{
     ApiClient,
     common::{config::paths::ensure_recovery_paths, lifecycle::spawn::configure_logging},
-    create_tempdir,
+    create_tempdir, persist_tempdir,
 };
 
 /// Minimal interface to apply common node setup.
@@ -137,10 +137,10 @@ where
         .spawn()
         .expect("failed to spawn node process");
 
-    let handle = NodeHandle::new(child, dir, config, ApiClient::new(addr, testing_addr));
+    let mut handle = NodeHandle::new(child, dir, config, ApiClient::new(addr, testing_addr));
 
     // Wait for readiness via consensus_info
-    time::timeout(Duration::from_secs(60), async {
+    let ready = time::timeout(Duration::from_secs(60), async {
         loop {
             if handle.api.consensus_info().await.is_ok() {
                 break;
@@ -148,7 +148,13 @@ where
             time::sleep(Duration::from_millis(100)).await;
         }
     })
-    .await?;
+    .await;
+
+    if let Err(err) = ready {
+        // Persist tempdir to aid debugging if readiness fails.
+        let _ = persist_tempdir(&mut handle.tempdir, "nomos-node");
+        return Err(err);
+    }
 
     info!("node readiness confirmed via consensus_info");
     Ok(handle)

testing-framework/core/src/scenario/cfgsync.rs

@@ -116,7 +116,6 @@ pub fn apply_topology_overrides(
     cfg.replication_settings = da.replication_settings;
     cfg.retry_shares_limit = da.retry_shares_limit;
     cfg.retry_commitments_limit = da.retry_commitments_limit;
-    cfg.tracing_settings = TracingSettings::default();
 }
 
 #[serde_as]

testing-framework/core/src/scenario/runtime/block_feed.rs

@@ -9,6 +9,7 @@ use std::{
 
 use anyhow::{Context as _, Result};
 use nomos_core::{block::Block, mantle::SignedMantleTx};
+use nomos_http_api_common::paths::STORAGE_BLOCK;
 use nomos_node::HeaderId;
 use tokio::{sync::broadcast, task::JoinHandle, time::sleep};
 use tracing::{debug, error};
@@ -108,7 +109,7 @@ impl BlockScanner {
     async fn run(&mut self) {
         loop {
             if let Err(err) = self.catch_up().await {
-                error!(%err, "block feed catch up failed");
+                error!(error = %err, error_debug = ?err, "block feed catch up failed");
             }
             sleep(POLL_INTERVAL).await;
         }
@@ -131,11 +132,22 @@ impl BlockScanner {
                 break;
             }
 
-            let block = self
+            let block = match self.client.storage_block(&cursor).await {
-                .client
+                Ok(block) => block,
-                .storage_block(&cursor)
+                Err(err) => {
-                .await?
+                    if err.is_decode() {
-                .context("missing block while catching up")?;
+                        if let Ok(resp) =
+                            self.client.post_json_response(STORAGE_BLOCK, &cursor).await
+                        {
+                            if let Ok(body) = resp.text().await {
+                                error!(header = ?cursor, %body, "failed to decode block response");
+                            }
+                        }
+                    }
+                    return Err(err.into());
+                }
+            }
+            .context("missing block while catching up")?;
 
             let parent = block.header().parent();
             stack.push((cursor, block));

testing-framework/core/src/topology/deployment.rs

@@ -181,11 +181,11 @@ impl Topology {
     fn node_listen_ports(&self) -> Vec<u16> {
         self.validators
             .iter()
-            .map(|node| node.config().network.backend.inner.port)
+            .map(|node| node.config().network.backend.swarm.port)
             .chain(
                 self.executors
                     .iter()
-                    .map(|node| node.config().network.backend.inner.port),
+                    .map(|node| node.config().network.backend.swarm.port),
             )
             .collect()
     }
@@ -221,13 +221,13 @@ impl Topology {
             .map(|(idx, node)| {
                 format!(
                     "validator#{idx}@{}",
-                    node.config().network.backend.inner.port
+                    node.config().network.backend.swarm.port
                 )
             })
             .chain(self.executors.iter().enumerate().map(|(idx, node)| {
                 format!(
                     "executor#{idx}@{}",
-                    node.config().network.backend.inner.port
+                    node.config().network.backend.swarm.port
                 )
             }))
             .collect()

testing-framework/core/src/topology/generation.rs

@@ -42,7 +42,7 @@ impl GeneratedNodeConfig {
 
     #[must_use]
     pub const fn network_port(&self) -> u16 {
-        self.general.network_config.backend.inner.port
+        self.general.network_config.backend.swarm.port
     }
 
     #[must_use]
@@ -218,11 +218,11 @@ impl GeneratedTopology {
     fn listen_ports(&self) -> Vec<u16> {
         self.validators
             .iter()
-            .map(|node| node.general.network_config.backend.inner.port)
+            .map(|node| node.general.network_config.backend.swarm.port)
             .chain(
                 self.executors
                     .iter()
-                    .map(|node| node.general.network_config.backend.inner.port),
+                    .map(|node| node.general.network_config.backend.swarm.port),
             )
             .collect()
     }
@@ -258,13 +258,13 @@ impl GeneratedTopology {
             .map(|(idx, node)| {
                 format!(
                     "validator#{idx}@{}",
-                    node.general.network_config.backend.inner.port
+                    node.general.network_config.backend.swarm.port
                 )
             })
             .chain(self.executors.iter().enumerate().map(|(idx, node)| {
                 format!(
                     "executor#{idx}@{}",
-                    node.general.network_config.backend.inner.port
+                    node.general.network_config.backend.swarm.port
                 )
             }))
             .collect()

testing-framework/core/src/topology/utils.rs

@@ -1,10 +1,7 @@
 use std::{collections::HashMap, iter};
 
 use groth16::fr_to_bytes;
-use key_management_system_service::{
+use key_management_system_service::{backend::preload::PreloadKMSBackendSettings, keys::Key};
-    backend::preload::PreloadKMSBackendSettings,
-    keys::{Ed25519Key, Key, ZkKey},
-};
 use nomos_utils::net::get_available_udp_port;
 use rand::{Rng, thread_rng};
 
@@ -25,31 +22,29 @@ pub fn create_kms_configs(
         .map(|(da_conf, blend_conf)| {
             let mut keys = HashMap::from([
                 (
-                    hex::encode(blend_conf.signer.verifying_key().as_bytes()),
+                    hex::encode(blend_conf.signer.public_key().to_bytes()),
-                    Key::Ed25519(Ed25519Key::new(blend_conf.signer.clone())),
+                    Key::Ed25519(blend_conf.signer.clone()),
                 ),
                 (
                     hex::encode(fr_to_bytes(
-                        &blend_conf.secret_zk_key.to_public_key().into_inner(),
+                        blend_conf.secret_zk_key.to_public_key().as_fr(),
                     )),
-                    Key::Zk(ZkKey::new(blend_conf.secret_zk_key.clone())),
+                    Key::Zk(blend_conf.secret_zk_key.clone()),
                 ),
                 (
-                    hex::encode(da_conf.signer.verifying_key().as_bytes()),
+                    hex::encode(da_conf.signer.public_key().to_bytes()),
-                    Key::Ed25519(Ed25519Key::new(da_conf.signer.clone())),
+                    Key::Ed25519(da_conf.signer.clone()),
                 ),
                 (
-                    hex::encode(fr_to_bytes(
+                    hex::encode(fr_to_bytes(da_conf.secret_zk_key.to_public_key().as_fr())),
-                        &da_conf.secret_zk_key.to_public_key().into_inner(),
+                    Key::Zk(da_conf.secret_zk_key.clone()),
-                    )),
-                    Key::Zk(ZkKey::new(da_conf.secret_zk_key.clone())),
                 ),
             ]);
 
             for account in wallet_accounts {
-                let key_id = hex::encode(fr_to_bytes(&account.public_key().into_inner()));
+                let key_id = hex::encode(fr_to_bytes(account.public_key().as_fr()));
                 keys.entry(key_id)
-                    .or_insert_with(|| Key::Zk(ZkKey::new(account.secret_key.clone())));
+                    .or_insert_with(|| Key::Zk(account.secret_key.clone()));
             }
 
             PreloadKMSBackendSettings { keys }

testing-framework/runners/compose/assets/docker-compose.yml.tera

@@ -13,6 +13,25 @@ services:
       - {{ prometheus.host_port }}
     restart: on-failure
 
+  grafana:
+    image: grafana/grafana:10.4.1
+    environment:
+      GF_PATHS_CONFIG: /etc/grafana/grafana.ini
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+    ports:
+      - {{ grafana.host_port }}
+    volumes:
+      - ./stack/monitoring/grafana/datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:z
+      - ./stack/monitoring/grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yaml:z
+      - ./stack/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - ./stack/monitoring/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
+    env_file:
+      - ./stack/monitoring/grafana/plugins.env
+    depends_on:
+      - prometheus
+    restart: on-failure
+
 {% for node in validators %}
   {{ node.name }}:
     image: {{ node.image }}
@@ -36,6 +55,11 @@ services:
 {% for env in node.environment %}
       {{ env.key }}: "{{ env.value }}"
 {% endfor %}
+    cap_add:
+      - SYS_ADMIN
+      - SYS_PTRACE
+    security_opt:
+      - seccomp=unconfined
     restart: on-failure
 
 {% endfor %}{% for node in executors %}
@@ -61,6 +85,11 @@ services:
 {% for env in node.environment %}
       {{ env.key }}: "{{ env.value }}"
 {% endfor %}
+    cap_add:
+      - SYS_ADMIN
+      - SYS_PTRACE
+    security_opt:
+      - seccomp=unconfined
     restart: on-failure
 
 {% endfor %}

testing-framework/runners/compose/src/deployer/orchestrator.rs

@@ -16,7 +16,10 @@ use super::{
 use crate::{
     docker::control::ComposeNodeControl,
     errors::ComposeRunnerError,
-    infrastructure::{environment::StackEnvironment, ports::compose_runner_host},
+    infrastructure::{
+        environment::StackEnvironment,
+        ports::{HostPortMapping, compose_runner_host},
+    },
     lifecycle::readiness::metrics_handle_from_port,
 };
 
@@ -71,6 +74,19 @@ impl DeploymentOrchestrator {
         let telemetry = metrics_handle_from_port(environment.prometheus_port(), &host)?;
         let node_control = self.maybe_node_control::<Caps>(&environment);
 
+        info!(
+            prometheus_url = %format!("http://{}:{}/", host, environment.prometheus_port()),
+            "prometheus endpoint available on host"
+        );
+        info!(
+            grafana_url = %format!("http://{}:{}/", host, environment.grafana_port()),
+            "grafana dashboard available on host"
+        );
+        log_profiling_urls(&host, &host_ports);
+
+        // Log profiling endpoints (profiling feature must be enabled in the binaries).
+        log_profiling_urls(&host, &host_ports);
+
         let (block_feed, block_feed_guard) = client_builder
             .start_block_feed(&node_clients, &mut environment)
             .await?;
@@ -113,3 +129,26 @@ impl DeploymentOrchestrator {
         })
     }
 }
+
+fn log_profiling_urls(host: &str, ports: &HostPortMapping) {
+    for (idx, node) in ports.validators.iter().enumerate() {
+        tracing::info!(
+            validator = idx,
+            profiling_url = %format!(
+                "http://{}:{}/debug/pprof/profile?seconds=15&format=proto",
+                host, node.api
+            ),
+            "validator profiling endpoint (profiling feature required)"
+        );
+    }
+    for (idx, node) in ports.executors.iter().enumerate() {
+        tracing::info!(
+            executor = idx,
+            profiling_url = %format!(
+                "http://{}:{}/debug/pprof/profile?seconds=15&format=proto",
+                host, node.api
+            ),
+            "executor profiling endpoint (profiling feature required)"
+        );
+    }
+}

testing-framework/runners/compose/src/descriptor/mod.rs

@@ -1,6 +1,6 @@
 use serde::Serialize;
 use testing_framework_core::{
-    constants::{DEFAULT_CFGSYNC_PORT, kzg_container_path},
+    constants::{DEFAULT_CFGSYNC_PORT, DEFAULT_PROMETHEUS_HTTP_PORT, kzg_container_path},
     topology::generation::{GeneratedNodeConfig, GeneratedTopology},
 };
 
@@ -21,6 +21,7 @@ pub enum DescriptorBuildError {
 #[derive(Clone, Debug, Serialize)]
 pub struct ComposeDescriptor {
     prometheus: PrometheusTemplate,
+    grafana: GrafanaTemplate,
     validators: Vec<NodeDescriptor>,
     executors: Vec<NodeDescriptor>,
 }
@@ -50,6 +51,7 @@ pub struct ComposeDescriptorBuilder<'a> {
     use_kzg_mount: bool,
     cfgsync_port: Option<u16>,
     prometheus_port: Option<u16>,
+    grafana_port: Option<u16>,
 }
 
 impl<'a> ComposeDescriptorBuilder<'a> {
@@ -59,6 +61,7 @@ impl<'a> ComposeDescriptorBuilder<'a> {
             use_kzg_mount: false,
             cfgsync_port: None,
             prometheus_port: None,
+            grafana_port: None,
         }
     }
 
@@ -83,12 +86,20 @@ impl<'a> ComposeDescriptorBuilder<'a> {
         self
     }
 
+    #[must_use]
+    /// Set host port mapping for Grafana.
+    pub const fn with_grafana_port(mut self, port: u16) -> Self {
+        self.grafana_port = Some(port);
+        self
+    }
+
     /// Finish building the descriptor, erroring if required fields are missing.
     pub fn build(self) -> Result<ComposeDescriptor, DescriptorBuildError> {
         let cfgsync_port = self.cfgsync_port.unwrap_or(DEFAULT_CFGSYNC_PORT);
         let prometheus_host_port = self
             .prometheus_port
             .ok_or(DescriptorBuildError::MissingPrometheusPort)?;
+        let grafana_host_port = self.grafana_port.unwrap_or(0);
 
         let (image, platform) = resolve_image();
         // Prometheus image is x86_64-only on some tags; set platform when on arm hosts.
@@ -117,6 +128,7 @@ impl<'a> ComposeDescriptorBuilder<'a> {
 
         Ok(ComposeDescriptor {
             prometheus: PrometheusTemplate::new(prometheus_host_port, prometheus_platform),
+            grafana: GrafanaTemplate::new(grafana_host_port),
             validators,
             executors,
         })
@@ -134,15 +146,29 @@ pub struct PrometheusTemplate {
 impl PrometheusTemplate {
     fn new(port: u16, platform: Option<String>) -> Self {
         Self {
-            host_port: format!(
+            host_port: format!("127.0.0.1:{port}:{}", DEFAULT_PROMETHEUS_HTTP_PORT),
-                "127.0.0.1:{port}:{}",
-                testing_framework_core::constants::DEFAULT_PROMETHEUS_HTTP_PORT
-            ),
             platform,
         }
     }
 }
 
+/// Minimal Grafana service mapping used in the compose template.
+#[derive(Clone, Debug, Serialize)]
+pub struct GrafanaTemplate {
+    host_port: String,
+}
+
+impl GrafanaTemplate {
+    fn new(port: u16) -> Self {
+        let host_port = match port {
+            0 => "127.0.0.1::3000".to_string(), // docker assigns host port
+            _ => format!("127.0.0.1:{port}:3000"),
+        };
+
+        Self { host_port }
+    }
+}
+
 #[derive(Clone, Copy)]
 pub(crate) enum ComposeNodeKind {
     Validator,

testing-framework/runners/compose/src/descriptor/node.rs

@@ -58,7 +58,7 @@ impl NodeDescriptor {
         environment.extend([
             EnvEntry::new(
                 "CFG_NETWORK_PORT",
-                node.general.network_config.backend.inner.port.to_string(),
+                node.general.network_config.backend.swarm.port.to_string(),
             ),
             EnvEntry::new("CFG_DA_PORT", node.da_port.to_string()),
             EnvEntry::new("CFG_BLEND_PORT", node.blend_port.to_string()),

testing-framework/runners/compose/src/infrastructure/environment.rs

@@ -1,14 +1,15 @@
 use std::{
+    env,
     net::{Ipv4Addr, TcpListener as StdTcpListener},
     path::{Path, PathBuf},
     time::Duration,
 };
 
-use anyhow::anyhow;
+use anyhow::{Context as _, anyhow};
 use testing_framework_core::{
     adjust_timeout, scenario::CleanupGuard, topology::generation::GeneratedTopology,
 };
-use tokio::process::Command;
+use tokio::{process::Command, time::timeout};
 use tracing::{debug, info, warn};
 use uuid::Uuid;
 
@@ -48,6 +49,7 @@ pub struct StackEnvironment {
     workspace: Option<ComposeWorkspace>,
     cfgsync_handle: Option<CfgsyncServerHandle>,
     prometheus_port: u16,
+    grafana_port: u16,
 }
 
 impl StackEnvironment {
@@ -58,6 +60,7 @@ impl StackEnvironment {
         project_name: String,
         cfgsync_handle: Option<CfgsyncServerHandle>,
         prometheus_port: u16,
+        grafana_port: u16,
     ) -> Self {
         let WorkspaceState {
             workspace, root, ..
@@ -70,6 +73,7 @@ impl StackEnvironment {
             workspace: Some(workspace),
             cfgsync_handle,
             prometheus_port,
+            grafana_port,
         }
     }
 
@@ -82,6 +86,11 @@ impl StackEnvironment {
         self.prometheus_port
     }
 
+    /// Host port exposed by Grafana.
+    pub const fn grafana_port(&self) -> u16 {
+        self.grafana_port
+    }
+
     /// Docker compose project name.
     pub fn project_name(&self) -> &str {
         &self.project_name
@@ -319,10 +328,12 @@ pub fn write_compose_artifacts(
     descriptors: &GeneratedTopology,
     cfgsync_port: u16,
     prometheus_port: u16,
+    grafana_port: u16,
 ) -> Result<PathBuf, ConfigError> {
     debug!(
         cfgsync_port,
         prometheus_port,
+        grafana_port,
         workspace_root = %workspace.root.display(),
         "building compose descriptor"
     );
@@ -330,6 +341,7 @@ pub fn write_compose_artifacts(
         .with_kzg_mount(workspace.use_kzg)
         .with_cfgsync_port(cfgsync_port)
         .with_prometheus_port(prometheus_port)
+        .with_grafana_port(grafana_port)
         .build()
         .map_err(|source| ConfigError::Descriptor { source })?;
 
@@ -346,13 +358,20 @@ pub fn render_compose_logged(
     descriptors: &GeneratedTopology,
     cfgsync_port: u16,
     prometheus_port: u16,
+    grafana_port: u16,
 ) -> Result<PathBuf, ComposeRunnerError> {
     info!(
         cfgsync_port,
-        prometheus_port, "rendering compose file with ports"
+        prometheus_port, grafana_port, "rendering compose file with ports"
     );
-    write_compose_artifacts(workspace, descriptors, cfgsync_port, prometheus_port)
+    write_compose_artifacts(
-        .map_err(Into::into)
+        workspace,
+        descriptors,
+        cfgsync_port,
+        prometheus_port,
+        grafana_port,
+    )
+    .map_err(Into::into)
 }
 
 /// Bring up docker compose; shut down cfgsync if start-up fails.
@@ -389,6 +408,12 @@ pub async fn prepare_environment(
 ) -> Result<StackEnvironment, ComposeRunnerError> {
     let workspace = prepare_workspace_logged()?;
     let cfgsync_port = allocate_cfgsync_port()?;
+    let grafana_env = env::var("COMPOSE_GRAFANA_PORT")
+        .ok()
+        .and_then(|raw| raw.parse::<u16>().ok());
+    if let Some(port) = grafana_env {
+        info!(port, "using grafana port from env");
+    }
     update_cfgsync_logged(&workspace, descriptors, cfgsync_port)?;
     ensure_compose_image().await?;
 
@@ -401,8 +426,14 @@ pub async fn prepare_environment(
 
     for _ in 0..attempts {
         let prometheus_port_value = prometheus_port.port();
-        let compose_path =
+        let grafana_port_value = grafana_env.unwrap_or(0);
-            render_compose_logged(&workspace, descriptors, cfgsync_port, prometheus_port_value)?;
+        let compose_path = render_compose_logged(
+            &workspace,
+            descriptors,
+            cfgsync_port,
+            prometheus_port_value,
+            grafana_port_value,
+        )?;
 
         let project_name = format!("nomos-compose-{}", Uuid::new_v4());
         let mut cfgsync_handle = start_cfgsync_stage(&workspace, cfgsync_port).await?;
@@ -417,11 +448,22 @@ pub async fn prepare_environment(
         .await
         {
             Ok(()) => {
+                let grafana_port_resolved = resolve_service_port(
+                    &compose_path,
+                    &project_name,
+                    &workspace.root,
+                    "grafana",
+                    3000,
+                )
+                .await
+                .unwrap_or(grafana_port_value);
+
                 info!(
                     project = %project_name,
                     compose_file = %compose_path.display(),
                     cfgsync_port,
                     prometheus_port = prometheus_port_value,
+                    grafana_port = grafana_port_resolved,
                     "compose stack is up"
                 );
                 return Ok(StackEnvironment::from_workspace(
@@ -430,6 +472,7 @@ pub async fn prepare_environment(
                     project_name,
                     Some(cfgsync_handle),
                     prometheus_port_value,
+                    grafana_port_resolved,
                 ));
             }
             Err(err) => {
@@ -466,3 +509,63 @@ fn reserve_prometheus_port(port: u16) -> Option<PortReservation> {
     let actual_port = listener.local_addr().ok()?.port();
     Some(PortReservation::new(actual_port, Some(listener)))
 }
+
+async fn resolve_service_port(
+    compose_file: &Path,
+    project_name: &str,
+    root: &Path,
+    service: &str,
+    container_port: u16,
+) -> Result<u16, ComposeRunnerError> {
+    let mut cmd = Command::new("docker");
+    cmd.arg("compose")
+        .arg("-f")
+        .arg(compose_file)
+        .arg("-p")
+        .arg(project_name)
+        .arg("port")
+        .arg(service)
+        .arg(container_port.to_string())
+        .current_dir(root);
+
+    let output = timeout(adjust_timeout(Duration::from_secs(30)), cmd.output())
+        .await
+        .map_err(|_| ComposeRunnerError::PortDiscovery {
+            service: service.to_owned(),
+            container_port,
+            source: anyhow!("docker compose port timed out"),
+        })?
+        .with_context(|| format!("running docker compose port {service} {container_port}"))
+        .map_err(|source| ComposeRunnerError::PortDiscovery {
+            service: service.to_owned(),
+            container_port,
+            source,
+        })?;
+
+    if !output.status.success() {
+        return Err(ComposeRunnerError::PortDiscovery {
+            service: service.to_owned(),
+            container_port,
+            source: anyhow!("docker compose port exited with {}", output.status),
+        });
+    }
+
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    for line in stdout.lines() {
+        let line = line.trim();
+        if line.is_empty() {
+            continue;
+        }
+        if let Some(port_str) = line.rsplit(':').next()
+            && let Ok(port) = port_str.trim().parse::<u16>()
+        {
+            return Ok(port);
+        }
+    }
+
+    Err(ComposeRunnerError::PortDiscovery {
+        service: service.to_owned(),
+        container_port,
+        source: anyhow!("unable to parse docker compose port output: {stdout}"),
+    })
+}

testing-framework/runners/k8s/helm/nomos-runner/grafana/dashboards/testnet_metrics.json

@@ -0,0 +1,237 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "da_mempool_pending_items",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Mempool: Pending DA blobs",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "barWidthFactor": 0.6,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 1,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "11.4.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "disableTextWrap": false,
+          "editorMode": "builder",
+          "expr": "consensus_processed_blocks",
+          "fullMetaSearch": false,
+          "includeNullMetadata": true,
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Consensus: Processed Blocks",
+      "type": "timeseries"
+    }
+  ],
+  "preload": false,
+  "schemaVersion": 40,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Testnet Metrics",
+  "uid": "ce6ebepwk737kf",
+  "version": 5,
+  "weekStart": ""
+}

testing-framework/runners/k8s/helm/nomos-runner/templates/_helpers.tpl

@@ -2,6 +2,10 @@
 {{- .Chart.Name -}}
 {{- end -}}
 
+{{- define "nomos-runner.name" -}}
+{{- include "nomos-runner.chart" . -}}
+{{- end -}}
+
 {{- define "nomos-runner.fullname" -}}
 {{- printf "%s" .Release.Name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}

testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-configmap.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.grafana.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nomos-runner.fullname" . }}-grafana-config
+data:
+  datasources.yaml: |
+    apiVersion: 1
+    datasources:
+      - name: Prometheus
+        type: prometheus
+        url: http://{{ include "nomos-runner.fullname" . }}-prometheus:9090
+        access: proxy
+        isDefault: true
+  dashboards.yaml: |
+    apiVersion: 1
+    providers:
+      - name: 'default'
+        orgId: 1
+        folder: 'Nomos'
+        type: file
+        disableDeletion: false
+        editable: true
+        options:
+          path: /var/lib/grafana/dashboards
+  testnet_metrics.json: |
+{{ (.Files.Get "grafana/dashboards/testnet_metrics.json") | indent 4 }}
+{{- end }}

testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-deployment.yaml

@@ -0,0 +1,62 @@
+{{- if .Values.grafana.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "nomos-runner.fullname" . }}-grafana
+  labels:
+    app: {{ include "nomos-runner.name" . }}
+    chart: {{ include "nomos-runner.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: {{ include "nomos-runner.name" . }}
+      component: grafana
+  template:
+    metadata:
+      labels:
+        app: {{ include "nomos-runner.name" . }}
+        component: grafana
+    spec:
+      containers:
+        - name: grafana
+          image: {{ .Values.grafana.image }}
+          imagePullPolicy: {{ .Values.grafana.imagePullPolicy }}
+          env:
+            - name: GF_SECURITY_ADMIN_USER
+              value: {{ .Values.grafana.adminUser | quote }}
+            - name: GF_SECURITY_ADMIN_PASSWORD
+              value: {{ .Values.grafana.adminPassword | quote }}
+          ports:
+            - containerPort: 3000
+              name: http
+          volumeMounts:
+            - name: grafana-config
+              mountPath: /etc/grafana/provisioning/datasources/datasources.yaml
+              subPath: datasources.yaml
+              readOnly: true
+            - name: grafana-config
+              mountPath: /etc/grafana/provisioning/dashboards/dashboards.yaml
+              subPath: dashboards.yaml
+              readOnly: true
+            - name: grafana-dashboards
+              mountPath: /var/lib/grafana/dashboards
+              readOnly: true
+      volumes:
+        - name: grafana-config
+          configMap:
+            name: {{ include "nomos-runner.fullname" . }}-grafana-config
+            items:
+              - key: datasources.yaml
+                path: datasources.yaml
+              - key: dashboards.yaml
+                path: dashboards.yaml
+        - name: grafana-dashboards
+          configMap:
+            name: {{ include "nomos-runner.fullname" . }}-grafana-config
+            items:
+              - key: testnet_metrics.json
+                path: testnet_metrics.json
+{{- end }}

testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-service.yaml

@@ -0,0 +1,22 @@
+{{- if .Values.grafana.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "nomos-runner.fullname" . }}-grafana
+  labels:
+    app: {{ include "nomos-runner.name" . }}
+    component: grafana
+spec:
+  type: {{ .Values.grafana.service.type }}
+  ports:
+    - port: 3000
+      targetPort: http
+      protocol: TCP
+      name: http
+      {{- if and (eq .Values.grafana.service.type "NodePort") .Values.grafana.service.nodePort }}
+      nodePort: {{ .Values.grafana.service.nodePort }}
+      {{- end }}
+  selector:
+    app: {{ include "nomos-runner.name" . }}
+    component: grafana
+{{- end }}

testing-framework/runners/k8s/helm/nomos-runner/values.yaml

@@ -37,3 +37,13 @@ prometheus:
       evaluation_interval: 15s
       external_labels:
         monitor: "NomosRunner"
+
+grafana:
+  enabled: true
+  image: "grafana/grafana:10.4.1"
+  imagePullPolicy: IfNotPresent
+  adminUser: admin
+  adminPassword: admin
+  service:
+    type: NodePort
+    nodePort: 30030

testing-framework/runners/k8s/src/deployer/orchestrator.rs

@@ -150,6 +150,11 @@ impl Deployer for K8sDeployer {
                 return Err(err);
             }
         };
+
+        tracing::info!(
+            grafana_url = %format!("http://{}:{}/", crate::host::node_host(), 30030),
+            "grafana dashboard available via NodePort"
+        );
         let (cleanup, port_forwards) = cluster
             .take()
             .expect("cluster should still be available")

testing-framework/runners/k8s/src/infrastructure/assets.rs

@@ -120,6 +120,7 @@ pub fn prepare_assets(topology: &GeneratedTopology) -> Result<RunnerAssets, Asse
 }
 
 const CFGSYNC_K8S_TIMEOUT_SECS: u64 = 300;
+const DEFAULT_GRAFANA_NODE_PORT: u16 = 30030;
 
 fn render_cfgsync_config(root: &Path, topology: &GeneratedTopology) -> Result<String, AssetsError> {
     let cfgsync_template_path = stack_assets_root(root).join("cfgsync.yaml");
@@ -243,6 +244,7 @@ struct HelmValues {
     cfgsync: CfgsyncValues,
     validators: NodeGroup,
     executors: NodeGroup,
+    grafana: GrafanaValues,
 }
 
 #[derive(Serialize)]
@@ -265,11 +267,43 @@ struct NodeValues {
     env: BTreeMap<String, String>,
 }
 
+#[derive(Serialize)]
+struct GrafanaValues {
+    enabled: bool,
+    image: String,
+    #[serde(rename = "imagePullPolicy")]
+    image_pull_policy: String,
+    #[serde(rename = "adminUser")]
+    admin_user: String,
+    #[serde(rename = "adminPassword")]
+    admin_password: String,
+    service: GrafanaServiceValues,
+}
+
+#[derive(Serialize)]
+struct GrafanaServiceValues {
+    #[serde(rename = "type")]
+    type_field: String,
+    #[serde(rename = "nodePort")]
+    node_port: Option<u16>,
+}
+
 fn build_values(topology: &GeneratedTopology) -> HelmValues {
     let cfgsync = CfgsyncValues {
         port: cfgsync_port(),
     };
     let pol_mode = pol_proof_mode();
+    let grafana = GrafanaValues {
+        enabled: true,
+        image: "grafana/grafana:10.4.1".into(),
+        image_pull_policy: "IfNotPresent".into(),
+        admin_user: "admin".into(),
+        admin_password: "admin".into(),
+        service: GrafanaServiceValues {
+            type_field: "NodePort".into(),
+            node_port: Some(DEFAULT_GRAFANA_NODE_PORT),
+        },
+    };
     debug!(pol_mode, "rendering Helm values for k8s stack");
     let validators = topology
         .validators()
@@ -355,6 +389,7 @@ fn build_values(topology: &GeneratedTopology) -> HelmValues {
             count: topology.executors().len(),
             nodes: executors,
         },
+        grafana,
     }
 }
 

testing-framework/tools/cfgsync/src/config/builder.rs

@@ -80,7 +80,7 @@ pub fn create_node_configs(
         .collect();
     let original_network_ports: Vec<u16> = network_configs
         .iter()
-        .map(|cfg| cfg.backend.inner.port)
+        .map(|cfg| cfg.backend.swarm.port)
         .collect();
     let peer_ids = build_peer_ids(&ids);
 
@@ -124,10 +124,10 @@ pub fn create_node_configs(
 
         // Libp2p network config.
         let mut network_config = network_configs[i].clone();
-        network_config.backend.inner.host = Ipv4Addr::from_str("0.0.0.0").unwrap();
+        network_config.backend.swarm.host = Ipv4Addr::from_str("0.0.0.0").unwrap();
-        network_config.backend.inner.port = host.network_port;
+        network_config.backend.swarm.port = host.network_port;
         network_config.backend.initial_peers = host_network_init_peers[i].clone();
-        network_config.backend.inner.nat_config = nomos_libp2p::NatSettings::Static {
+        network_config.backend.swarm.nat_config = nomos_libp2p::NatSettings::Static {
             external_address: Multiaddr::from_str(&format!(
                 "/ip4/{}/udp/{}/quic-v1",
                 host.ip, host.network_port

testing-framework/tools/cfgsync/src/config/kms.rs

@@ -1,8 +1,5 @@
 use groth16::fr_to_bytes;
-use key_management_system_service::{
+use key_management_system_service::{backend::preload::PreloadKMSBackendSettings, keys::Key};
-    backend::preload::PreloadKMSBackendSettings,
-    keys::{Ed25519Key, Key, ZkKey},
-};
 use testing_framework_config::topology::configs::{blend::GeneralBlendConfig, da::GeneralDaConfig};
 
 pub fn create_kms_configs(
@@ -15,24 +12,22 @@ pub fn create_kms_configs(
         .map(|(da_conf, blend_conf)| PreloadKMSBackendSettings {
             keys: [
                 (
-                    hex::encode(blend_conf.signer.verifying_key().as_bytes()),
+                    hex::encode(blend_conf.signer.public_key().to_bytes()),
-                    Key::Ed25519(Ed25519Key::new(blend_conf.signer.clone())),
+                    Key::Ed25519(blend_conf.signer.clone()),
                 ),
                 (
                     hex::encode(fr_to_bytes(
-                        &blend_conf.secret_zk_key.to_public_key().into_inner(),
+                        blend_conf.secret_zk_key.to_public_key().as_fr(),
                     )),
-                    Key::Zk(ZkKey::new(blend_conf.secret_zk_key.clone())),
+                    Key::Zk(blend_conf.secret_zk_key.clone()),
                 ),
                 (
-                    hex::encode(da_conf.signer.verifying_key().as_bytes()),
+                    hex::encode(da_conf.signer.public_key().to_bytes()),
-                    Key::Ed25519(Ed25519Key::new(da_conf.signer.clone())),
+                    Key::Ed25519(da_conf.signer.clone()),
                 ),
                 (
-                    hex::encode(fr_to_bytes(
+                    hex::encode(fr_to_bytes(da_conf.secret_zk_key.to_public_key().as_fr())),
-                        &da_conf.secret_zk_key.to_public_key().into_inner(),
+                    Key::Zk(da_conf.secret_zk_key.clone()),
-                    )),
-                    Key::Zk(ZkKey::new(da_conf.secret_zk_key.clone())),
                 ),
             ]
             .into(),

testing-framework/workflows/Cargo.toml

@@ -13,18 +13,19 @@ version               = "0.1.0"
 workspace = true
 
 [dependencies]
-async-trait              = "0.1"
+async-trait                     = "0.1"
-chain-service            = { git = "https://github.com/logos-co/nomos-node", rev = "d2dd5a5084e1daef4032562c77d41de5e4d495f8" }
+chain-service                   = { workspace = true }
-ed25519-dalek            = { version = "2.2.0", features = ["rand_core", "serde"] }
+ed25519-dalek                   = { version = "2.2.0", features = ["rand_core", "serde"] }
-executor-http-client     = { workspace = true }
+executor-http-client            = { workspace = true }
-nomos-core               = { workspace = true }
+key-management-system-service   = { workspace = true }
-rand                     = { workspace = true }
+nomos-core                      = { workspace = true }
-testing-framework-config = { workspace = true }
+rand                            = { workspace = true }
-testing-framework-core   = { workspace = true }
+testing-framework-config        = { workspace = true }
-thiserror                = { workspace = true }
+testing-framework-core          = { workspace = true }
-tokio                    = { workspace = true, features = ["macros", "net", "rt-multi-thread", "time"] }
+thiserror                       = { workspace = true }
-tracing                  = { workspace = true }
+futures                         = "0.3"
-zksign                   = { workspace = true }
+tokio                           = { workspace = true, features = ["macros", "net", "rt-multi-thread", "time"] }
+tracing                         = { workspace = true }
 
 [package.metadata.cargo-machete]
 ignored = ["chain-service"]

testing-framework/workflows/src/builder/mod.rs

@@ -158,6 +158,7 @@ pub struct DataAvailabilityFlowBuilder<Caps> {
     builder: CoreScenarioBuilder<Caps>,
     channel_rate: NonZeroU64,
     blob_rate: NonZeroU64,
+    headroom_percent: u64,
 }
 
 impl<Caps> DataAvailabilityFlowBuilder<Caps> {
@@ -174,18 +175,19 @@ impl<Caps> DataAvailabilityFlowBuilder<Caps> {
             builder,
             channel_rate: Self::default_channel_rate(),
             blob_rate: Self::default_blob_rate(),
+            headroom_percent: da::Workload::default_headroom_percent(),
         }
     }
 
     #[must_use]
-    /// Set channel publish rate per block (panics on zero).
+    /// Set the number of DA channels to run (panics on zero).
     pub const fn channel_rate(mut self, rate: u64) -> Self {
         self.channel_rate = channel_rate_checked(rate);
         self
     }
 
     #[must_use]
-    /// Set channel publish rate per block.
+    /// Set the number of DA channels to run.
     pub const fn channel_rate_per_block(mut self, rate: NonZeroU64) -> Self {
         self.channel_rate = rate;
         self
@@ -205,14 +207,21 @@ impl<Caps> DataAvailabilityFlowBuilder<Caps> {
         self
     }
 
+    #[must_use]
+    /// Apply headroom when converting blob rate into channel count.
+    pub const fn headroom_percent(mut self, percent: u64) -> Self {
+        self.headroom_percent = percent;
+        self
+    }
+
     #[must_use]
     pub fn apply(mut self) -> CoreScenarioBuilder<Caps> {
-        let count = (self.channel_rate.get() * self.blob_rate.get()) as usize;
+        let workload =
-        let workload = da::Workload::with_channel_count(count.max(1));
+            da::Workload::with_rate(self.blob_rate, self.channel_rate, self.headroom_percent);
         tracing::info!(
             channel_rate = self.channel_rate.get(),
             blob_rate = self.blob_rate.get(),
-            channels = count.max(1),
+            headroom_percent = self.headroom_percent,
             "attaching data-availability workload"
         );
         self.builder = self.builder.with_workload(workload);

testing-framework/workflows/src/util/tx.rs

@@ -1,17 +1,16 @@
-use ed25519_dalek::{Signer as _, SigningKey};
+use key_management_system_service::keys::{Ed25519Key, ZkKey};
 use nomos_core::mantle::{
     MantleTx, Op, OpProof, SignedMantleTx, Transaction as _,
     ledger::Tx as LedgerTx,
     ops::channel::{ChannelId, MsgId, inscribe::InscriptionOp},
 };
-use zksign::SecretKey;
 
 /// Builds a signed inscription transaction with deterministic payload for
 /// testing.
 #[must_use]
 pub fn create_inscription_transaction_with_id(id: ChannelId) -> SignedMantleTx {
-    let signing_key = SigningKey::from_bytes(&[0u8; 32]);
+    let signing_key = Ed25519Key::from_bytes(&[0u8; 32]);
-    let signer = signing_key.verifying_key();
+    let signer = signing_key.public_key();
 
     let inscription_op = InscriptionOp {
         channel_id: id,
@@ -28,13 +27,14 @@ pub fn create_inscription_transaction_with_id(id: ChannelId) -> SignedMantleTx {
     };
 
     let tx_hash = mantle_tx.hash();
-    let signature = signing_key.sign(&tx_hash.as_signing_bytes());
+    let signature = signing_key.sign_payload(tx_hash.as_signing_bytes().as_ref());
+    let zk_key = ZkKey::zero();
     tracing::debug!(channel = ?id, tx_hash = ?tx_hash, "building inscription transaction");
 
     SignedMantleTx::new(
         mantle_tx,
         vec![OpProof::Ed25519Sig(signature)],
-        SecretKey::multi_sign(&[], tx_hash.as_ref()).expect("zk signature generation"),
+        ZkKey::multi_sign(&[zk_key], tx_hash.as_ref()).expect("zk signature generation"),
     )
     .expect("valid transaction")
 }

testing-framework/workflows/src/workloads/da/expectation.rs

@@ -1,5 +1,6 @@
 use std::{
-    collections::HashSet,
+    collections::{HashMap, HashSet},
+    num::NonZeroU64,
     sync::{Arc, Mutex},
 };
 
@@ -12,9 +13,13 @@ use testing_framework_core::scenario::{BlockRecord, DynError, Expectation, RunCo
 use thiserror::Error;
 use tokio::sync::broadcast;
 
+use super::workload::{planned_blob_count, planned_channel_count, planned_channel_ids};
+
 #[derive(Debug)]
 pub struct DaWorkloadExpectation {
-    planned_channels: Vec<ChannelId>,
+    blob_rate_per_block: NonZeroU64,
+    channel_rate_per_block: NonZeroU64,
+    headroom_percent: u64,
     capture_state: Option<CaptureState>,
 }
 
@@ -22,7 +27,8 @@ pub struct DaWorkloadExpectation {
 struct CaptureState {
     planned: Arc<HashSet<ChannelId>>,
     inscriptions: Arc<Mutex<HashSet<ChannelId>>>,
-    blobs: Arc<Mutex<HashSet<ChannelId>>>,
+    blobs: Arc<Mutex<HashMap<ChannelId, u64>>>,
+    expected_total_blobs: u64,
 }
 
 const MIN_INCLUSION_RATIO: f64 = 0.8;
@@ -39,9 +45,15 @@ enum DaExpectationError {
 
 impl DaWorkloadExpectation {
     /// Validates that inscriptions and blobs landed for the planned channels.
-    pub const fn new(planned_channels: Vec<ChannelId>) -> Self {
+    pub const fn new(
+        blob_rate_per_block: NonZeroU64,
+        channel_rate_per_block: NonZeroU64,
+        headroom_percent: u64,
+    ) -> Self {
         Self {
-            planned_channels,
+            blob_rate_per_block,
+            channel_rate_per_block,
+            headroom_percent,
             capture_state: None,
         }
     }
@@ -58,19 +70,24 @@ impl Expectation for DaWorkloadExpectation {
             return Ok(());
         }
 
+        let planned_ids = planned_channel_ids(planned_channel_count(
+            self.channel_rate_per_block,
+            self.headroom_percent,
+        ));
+
+        let expected_total_blobs = planned_blob_count(self.blob_rate_per_block, &ctx.run_metrics());
+
         tracing::info!(
-            planned_channels = self.planned_channels.len(),
+            planned_channels = planned_ids.len(),
+            blob_rate_per_block = self.blob_rate_per_block.get(),
+            headroom_percent = self.headroom_percent,
+            expected_total_blobs,
             "DA inclusion expectation starting capture"
         );
 
-        let planned = Arc::new(
+        let planned = Arc::new(planned_ids.iter().copied().collect::<HashSet<_>>());
-            self.planned_channels
-                .iter()
-                .copied()
-                .collect::<HashSet<_>>(),
-        );
         let inscriptions = Arc::new(Mutex::new(HashSet::new()));
-        let blobs = Arc::new(Mutex::new(HashSet::new()));
+        let blobs = Arc::new(Mutex::new(HashMap::new()));
 
         let mut receiver = ctx.block_feed().subscribe();
         let planned_for_task = Arc::clone(&planned);
@@ -101,6 +118,7 @@ impl Expectation for DaWorkloadExpectation {
             planned,
             inscriptions,
             blobs,
+            expected_total_blobs,
         });
 
         Ok(())
@@ -135,20 +153,20 @@ impl Expectation for DaWorkloadExpectation {
             .into());
         }
 
-        let missing_blobs = {
+        let observed_total_blobs = {
             let blobs = state.blobs.lock().expect("blob lock poisoned");
-            missing_channels(&state.planned, &blobs)
+            blobs.values().sum::<u64>()
         };
-        let required_blobs = minimum_required(planned_total, MIN_INCLUSION_RATIO);
+        let required_blobs = minimum_required_u64(state.expected_total_blobs, MIN_INCLUSION_RATIO);
-        if planned_total.saturating_sub(missing_blobs.len()) < required_blobs {
+        if observed_total_blobs < required_blobs {
             tracing::warn!(
-                planned = planned_total,
+                planned = state.expected_total_blobs,
-                missing = missing_blobs.len(),
+                observed = observed_total_blobs,
                 required = required_blobs,
                 "DA expectation missing blobs"
             );
             return Err(DaExpectationError::MissingBlobs {
-                missing: missing_blobs,
+                missing: Vec::new(),
             }
             .into());
         }
@@ -156,7 +174,7 @@ impl Expectation for DaWorkloadExpectation {
         tracing::info!(
             planned = planned_total,
             inscriptions = planned_total - missing_inscriptions.len(),
-            blobs = planned_total - missing_blobs.len(),
+            blobs_observed = observed_total_blobs,
             "DA inclusion expectation satisfied"
         );
 
@@ -168,7 +186,7 @@ fn capture_block(
     block: &BlockRecord,
     planned: &HashSet<ChannelId>,
     inscriptions: &Arc<Mutex<HashSet<ChannelId>>>,
-    blobs: &Arc<Mutex<HashSet<ChannelId>>>,
+    blobs: &Arc<Mutex<HashMap<ChannelId, u64>>>,
 ) {
     let mut new_inscriptions = Vec::new();
     let mut new_blobs = Vec::new();
@@ -195,8 +213,14 @@ fn capture_block(
 
     if !new_blobs.is_empty() {
         let mut guard = blobs.lock().expect("blob lock poisoned");
-        guard.extend(new_blobs);
+        for channel in new_blobs {
-        tracing::debug!(count = guard.len(), "DA expectation captured blobs");
+            let entry = guard.entry(channel).or_insert(0);
+            *entry += 1;
+        }
+        tracing::debug!(
+            total_blobs = guard.values().sum::<u64>(),
+            "DA expectation captured blobs"
+        );
     }
 }
 
@@ -207,3 +231,7 @@ fn missing_channels(planned: &HashSet<ChannelId>, observed: &HashSet<ChannelId>)
 fn minimum_required(total: usize, ratio: f64) -> usize {
     ((total as f64) * ratio).ceil() as usize
 }
+
+fn minimum_required_u64(total: u64, ratio: f64) -> u64 {
+    ((total as f64) * ratio).ceil() as u64
+}

testing-framework/workflows/src/workloads/da/workload.rs

@@ -1,8 +1,10 @@
-use std::{sync::Arc, time::Duration};
+use std::{num::NonZeroU64, sync::Arc, time::Duration};
 
 use async_trait::async_trait;
 use ed25519_dalek::SigningKey;
 use executor_http_client::ExecutorHttpClient;
+use futures::future::try_join_all;
+use key_management_system_service::keys::Ed25519PublicKey;
 use nomos_core::{
     da::BlobId,
     mantle::ops::{
@@ -13,7 +15,9 @@ use nomos_core::{
 use rand::{Rng as _, RngCore as _, seq::SliceRandom as _, thread_rng};
 use testing_framework_core::{
     nodes::ApiClient,
-    scenario::{BlockRecord, DynError, Expectation, RunContext, Workload as ScenarioWorkload},
+    scenario::{
+        BlockRecord, DynError, Expectation, RunContext, RunMetrics, Workload as ScenarioWorkload,
+    },
 };
 use tokio::{sync::broadcast, time::sleep};
 
@@ -24,36 +28,50 @@ use crate::{
 };
 
 const TEST_KEY_BYTES: [u8; 32] = [0u8; 32];
-const DEFAULT_CHANNELS: usize = 1;
+const DEFAULT_BLOB_RATE_PER_BLOCK: u64 = 1;
+const DEFAULT_CHANNEL_RATE_PER_BLOCK: u64 = 1;
 const MIN_BLOB_CHUNKS: usize = 1;
 const MAX_BLOB_CHUNKS: usize = 8;
 const PUBLISH_RETRIES: usize = 5;
 const PUBLISH_RETRY_DELAY: Duration = Duration::from_secs(2);
+const DEFAULT_HEADROOM_PERCENT: u64 = 20;
 
 #[derive(Clone)]
 pub struct Workload {
-    planned_channels: Arc<[ChannelId]>,
+    blob_rate_per_block: NonZeroU64,
+    channel_rate_per_block: NonZeroU64,
+    headroom_percent: u64,
 }
 
 impl Default for Workload {
     fn default() -> Self {
-        Self::with_channel_count(DEFAULT_CHANNELS)
+        Self::with_rate(
+            NonZeroU64::new(DEFAULT_BLOB_RATE_PER_BLOCK).expect("non-zero"),
+            NonZeroU64::new(DEFAULT_CHANNEL_RATE_PER_BLOCK).expect("non-zero"),
+            DEFAULT_HEADROOM_PERCENT,
+        )
     }
 }
 
 impl Workload {
-    /// Creates a workload that inscribes and publishes blobs on `count`
+    /// Creates a workload that targets a blobs-per-block rate and applies a
-    /// channels.
+    /// headroom factor when deriving the channel count.
     #[must_use]
-    pub fn with_channel_count(count: usize) -> Self {
+    pub const fn with_rate(
-        assert!(count > 0, "da workload requires positive count");
+        blob_rate_per_block: NonZeroU64,
+        channel_rate_per_block: NonZeroU64,
+        headroom_percent: u64,
+    ) -> Self {
         Self {
-            planned_channels: Arc::from(planned_channel_ids(count)),
+            blob_rate_per_block,
+            channel_rate_per_block,
+            headroom_percent,
         }
     }
 
-    fn plan(&self) -> Arc<[ChannelId]> {
+    #[must_use]
-        Arc::clone(&self.planned_channels)
+    pub const fn default_headroom_percent() -> u64 {
+        DEFAULT_HEADROOM_PERCENT
     }
 }
 
@@ -64,18 +82,44 @@ impl ScenarioWorkload for Workload {
     }
 
     fn expectations(&self) -> Vec<Box<dyn Expectation>> {
-        let planned = self.plan().to_vec();
+        vec![Box::new(DaWorkloadExpectation::new(
-        vec![Box::new(DaWorkloadExpectation::new(planned))]
+            self.blob_rate_per_block,
+            self.channel_rate_per_block,
+            self.headroom_percent,
+        ))]
     }
 
     async fn start(&self, ctx: &RunContext) -> Result<(), DynError> {
-        let mut receiver = ctx.block_feed().subscribe();
+        let planned_channels = planned_channel_ids(planned_channel_count(
+            self.channel_rate_per_block,
+            self.headroom_percent,
+        ));
 
-        for channel_id in self.plan().iter().copied() {
+        let expected_blobs = planned_blob_count(self.blob_rate_per_block, &ctx.run_metrics());
-            tracing::info!(channel_id = ?channel_id, "DA workload starting channel flow");
+        let per_channel_target =
-            run_channel_flow(ctx, &mut receiver, channel_id).await?;
+            per_channel_blob_target(expected_blobs, planned_channels.len().max(1) as u64);
-            tracing::info!(channel_id = ?channel_id, "DA workload finished channel flow");
+
-        }
+        tracing::info!(
+            blob_rate_per_block = self.blob_rate_per_block.get(),
+            channel_rate = self.channel_rate_per_block.get(),
+            headroom_percent = self.headroom_percent,
+            planned_channels = planned_channels.len(),
+            expected_blobs,
+            per_channel_target,
+            "DA workload derived planned channels"
+        );
+
+        try_join_all(planned_channels.into_iter().map(|channel_id| {
+            let ctx = ctx;
+            async move {
+                let mut receiver = ctx.block_feed().subscribe();
+                tracing::info!(channel_id = ?channel_id, blobs = per_channel_target, "DA workload starting channel flow");
+                run_channel_flow(ctx, &mut receiver, channel_id, per_channel_target).await?;
+                tracing::info!(channel_id = ?channel_id, "DA workload finished channel flow");
+                Ok::<(), DynError>(())
+            }
+        }))
+        .await?;
 
         tracing::info!("DA workload completed all channel flows");
         Ok(())
@@ -86,6 +130,7 @@ async fn run_channel_flow(
     ctx: &RunContext,
     receiver: &mut broadcast::Receiver<Arc<BlockRecord>>,
     channel_id: ChannelId,
+    target_blobs: u64,
 ) -> Result<(), DynError> {
     tracing::debug!(channel_id = ?channel_id, "DA: submitting inscription tx");
     let tx = Arc::new(tx::create_inscription_transaction_with_id(channel_id));
@@ -93,9 +138,13 @@ async fn run_channel_flow(
 
     let inscription_id = wait_for_inscription(receiver, channel_id).await?;
     tracing::debug!(channel_id = ?channel_id, inscription_id = ?inscription_id, "DA: inscription observed");
-    let blob_id = publish_blob(ctx, channel_id, inscription_id).await?;
+    let mut parent_id = inscription_id;
-    tracing::debug!(channel_id = ?channel_id, blob_id = ?blob_id, "DA: blob published");
+
-    wait_for_blob(receiver, channel_id, blob_id).await?;
+    for _ in 0..target_blobs {
+        let blob_id = publish_blob(ctx, channel_id, parent_id).await?;
+        tracing::debug!(channel_id = ?channel_id, blob_id = ?blob_id, "DA: blob published");
+        parent_id = wait_for_blob(receiver, channel_id, blob_id).await?;
+    }
     Ok(())
 }
 
@@ -166,7 +215,9 @@ async fn publish_blob(
         return Err("da workload requires at least one executor".into());
     }
 
-    let signer = SigningKey::from_bytes(&TEST_KEY_BYTES).verifying_key();
+    let signer: Ed25519PublicKey = SigningKey::from_bytes(&TEST_KEY_BYTES)
+        .verifying_key()
+        .into();
     let data = random_blob_payload();
     tracing::debug!(channel = ?channel_id, payload_bytes = data.len(), "DA: prepared blob payload");
     let client = ExecutorHttpClient::new(None);
@@ -205,7 +256,7 @@ fn random_blob_payload() -> Vec<u8> {
     data
 }
 
-fn planned_channel_ids(total: usize) -> Vec<ChannelId> {
+pub fn planned_channel_ids(total: usize) -> Vec<ChannelId> {
     (0..total as u64)
         .map(deterministic_channel_id)
         .collect::<Vec<_>>()
@@ -217,3 +268,26 @@ fn deterministic_channel_id(index: u64) -> ChannelId {
     bytes[24..].copy_from_slice(&index.to_be_bytes());
     ChannelId::from(bytes)
 }
+
+#[must_use]
+pub fn planned_channel_count(channel_rate_per_block: NonZeroU64, headroom_percent: u64) -> usize {
+    let base = channel_rate_per_block.get() as usize;
+    let extra = (base.saturating_mul(headroom_percent as usize) + 99) / 100;
+    let total = base.saturating_add(extra);
+    total.max(1)
+}
+
+#[must_use]
+pub fn planned_blob_count(blob_rate_per_block: NonZeroU64, run_metrics: &RunMetrics) -> u64 {
+    let expected_blocks = run_metrics.expected_consensus_blocks().max(1);
+    blob_rate_per_block.get().saturating_mul(expected_blocks)
+}
+
+#[must_use]
+pub fn per_channel_blob_target(total_blobs: u64, channel_count: u64) -> u64 {
+    if channel_count == 0 {
+        return total_blobs.max(1);
+    }
+    let per = (total_blobs + channel_count - 1) / channel_count;
+    per.max(1)
+}

testing-framework/workflows/src/workloads/transaction/expectation.rs

@@ -8,11 +8,11 @@ use std::{
 };
 
 use async_trait::async_trait;
+use key_management_system_service::keys::ZkPublicKey;
 use nomos_core::{header::HeaderId, mantle::AuthenticatedMantleTx as _};
 use testing_framework_core::scenario::{DynError, Expectation, RunContext};
 use thiserror::Error;
 use tokio::sync::broadcast;
-use zksign::PublicKey;
 
 use super::workload::{limited_user_count, submission_plan};
 
@@ -93,12 +93,12 @@ impl Expectation for TxInclusionExpectation {
             .into_iter()
             .take(planned)
             .map(|account| account.secret_key.to_public_key())
-            .collect::<HashSet<PublicKey>>();
+            .collect::<HashSet<ZkPublicKey>>();
 
         let observed = Arc::new(AtomicU64::new(0));
         let receiver = ctx.block_feed().subscribe();
-        let tracked_accounts = Arc::new(wallet_pks);
+        let tracked_accounts: Arc<HashSet<ZkPublicKey>> = Arc::new(wallet_pks);
-        let spawn_accounts = Arc::clone(&tracked_accounts);
+        let spawn_accounts: Arc<HashSet<ZkPublicKey>> = Arc::clone(&tracked_accounts);
         let spawn_observed = Arc::clone(&observed);
 
         tokio::spawn(async move {

testing-framework/workflows/src/workloads/transaction/workload.rs

@@ -6,6 +6,7 @@ use std::{
 };
 
 use async_trait::async_trait;
+use key_management_system_service::keys::{ZkKey, ZkPublicKey};
 use nomos_core::mantle::{
     GenesisTx as _, Note, SignedMantleTx, Transaction as _, Utxo, tx_builder::MantleTxBuilder,
 };
@@ -15,7 +16,6 @@ use testing_framework_core::{
     topology::generation::{GeneratedNodeConfig, GeneratedTopology},
 };
 use tokio::time::sleep;
-use zksign::{PublicKey, SecretKey};
 
 use super::expectation::TxInclusionExpectation;
 use crate::workloads::util::submit_transaction_via_cluster;
@@ -214,7 +214,7 @@ fn build_wallet_transaction(input: &WalletInput) -> Result<SignedMantleTx, DynEr
     let mantle_tx = builder.build();
     let tx_hash = mantle_tx.hash();
 
-    let signature = SecretKey::multi_sign(
+    let signature = ZkKey::multi_sign(
         std::slice::from_ref(&input.account.secret_key),
         tx_hash.as_ref(),
     )
@@ -225,7 +225,7 @@ fn build_wallet_transaction(input: &WalletInput) -> Result<SignedMantleTx, DynEr
     })
 }
 
-fn wallet_utxo_map(node: &GeneratedNodeConfig) -> HashMap<PublicKey, Utxo> {
+fn wallet_utxo_map(node: &GeneratedNodeConfig) -> HashMap<ZkPublicKey, Utxo> {
     let genesis_tx = node.general.consensus_config.genesis_tx.clone();
     let ledger_tx = genesis_tx.mantle_tx().ledger_tx.clone();
     let tx_hash = ledger_tx.hash();

versions.env

@@ -1,3 +1,4 @@
 VERSION=v0.3.1
-NOMOS_NODE_REV=d2dd5a5084e1daef4032562c77d41de5e4d495f8
 NOMOS_BUNDLE_VERSION=v4
+NOMOS_NODE_PATH=/Users/andrussalumets/IdeaProjects/nomos-node
+# NOMOS_NODE_REV=365b36469c34a19c607eed7d8f7a3e5380dac36e