diff --git a/Cargo.lock b/Cargo.lock index 97c6194..6cc26e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -836,7 +836,7 @@ dependencies = [ [[package]] name = "broadcast-service" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "derivative", @@ -995,7 +995,7 @@ dependencies = [ [[package]] name = "chain-common" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "nomos-core", "serde", @@ -1004,7 +1004,7 @@ dependencies = [ [[package]] name = "chain-leader" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "chain-common", @@ -1031,7 +1031,7 @@ dependencies = [ [[package]] name = "chain-network" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "chain-common", @@ -1060,7 +1060,7 @@ dependencies = [ [[package]] name = "chain-service" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "broadcast-service", @@ -1133,7 +1133,7 @@ dependencies = [ [[package]] name = "circuits-prover" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "circuits-utils", "tempfile", @@ -1142,7 +1142,7 @@ dependencies = [ [[package]] name = "circuits-utils" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "dirs", ] @@ -1212,7 +1212,7 @@ dependencies = [ [[package]] name = "common-http-client" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "broadcast-service", "futures", @@ -1363,7 +1363,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "cryptarchia-engine" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "cfg_eval", "nomos-utils", @@ -1378,7 +1378,7 @@ dependencies = [ [[package]] name = "cryptarchia-sync" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "bytes", "cryptarchia-engine", @@ -1866,7 +1866,7 @@ dependencies = [ [[package]] name = "executor-http-client" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "common-http-client", "futures", @@ -1874,6 +1874,8 @@ dependencies = [ "nomos-http-api-common", "reqwest", "serde", + "serde_json", + "tracing", ] [[package]] @@ -2246,7 +2248,7 @@ dependencies = [ [[package]] name = "groth16" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "ark-bn254 0.4.0", "ark-ec 0.4.2", @@ -3142,7 +3144,7 @@ dependencies = [ [[package]] name = "key-management-system-keys" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "bytes", @@ -3167,7 +3169,7 @@ dependencies = [ [[package]] name = "key-management-system-macros" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "proc-macro2", "quote", @@ -3177,7 +3179,7 @@ dependencies = [ [[package]] name = "key-management-system-service" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "key-management-system-keys", @@ -3283,7 +3285,7 @@ dependencies = [ [[package]] name = "kzgrs" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "ark-bls12-381", "ark-ec 0.4.2", @@ -3302,7 +3304,7 @@ dependencies = [ [[package]] name = "kzgrs-backend" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "ark-ff 0.4.2", "ark-poly 0.4.2", @@ -3926,7 +3928,7 @@ dependencies = [ [[package]] name = "mmr" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "ark-ff 0.4.2", "groth16", @@ -4213,7 +4215,7 @@ dependencies = [ [[package]] name = "nomos-api" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "broadcast-service", @@ -4246,7 +4248,7 @@ dependencies = [ [[package]] name = "nomos-blend" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "nomos-blend-crypto", "nomos-blend-message", @@ -4258,7 +4260,7 @@ dependencies = [ [[package]] name = "nomos-blend-crypto" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "blake2", "groth16", @@ -4276,7 +4278,7 @@ dependencies = [ [[package]] name = "nomos-blend-message" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "blake2", "derivative", @@ -4298,7 +4300,7 @@ dependencies = [ [[package]] name = "nomos-blend-network" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "either", "futures", @@ -4316,7 +4318,7 @@ dependencies = [ [[package]] name = "nomos-blend-proofs" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "ed25519-dalek", "generic-array 1.3.5", @@ -4331,7 +4333,7 @@ dependencies = [ [[package]] name = "nomos-blend-scheduling" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "derivative", @@ -4355,7 +4357,7 @@ dependencies = [ [[package]] name = "nomos-blend-service" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "broadcast-service", @@ -4390,7 +4392,7 @@ dependencies = [ [[package]] name = "nomos-core" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "ark-ff 0.4.2", "async-trait", @@ -4420,7 +4422,7 @@ dependencies = [ [[package]] name = "nomos-da-dispersal" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "broadcast-service", @@ -4446,7 +4448,7 @@ dependencies = [ [[package]] name = "nomos-da-messages" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "blake2", "futures", @@ -4459,7 +4461,7 @@ dependencies = [ [[package]] name = "nomos-da-network-core" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "cached", "fixed", @@ -4486,7 +4488,7 @@ dependencies = [ [[package]] name = "nomos-da-network-service" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "arc-swap", "async-trait", @@ -4523,7 +4525,7 @@ dependencies = [ [[package]] name = "nomos-da-sampling" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "bytes", @@ -4551,7 +4553,7 @@ dependencies = [ [[package]] name = "nomos-da-verifier" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "futures", @@ -4578,7 +4580,7 @@ dependencies = [ [[package]] name = "nomos-executor" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "axum", @@ -4617,7 +4619,7 @@ dependencies = [ [[package]] name = "nomos-http-api-common" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "axum", "governor", @@ -4627,12 +4629,13 @@ dependencies = [ "serde_json", "serde_with", "tower_governor", + "tracing", ] [[package]] name = "nomos-ledger" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "cryptarchia-engine", "groth16", @@ -4655,7 +4658,7 @@ dependencies = [ [[package]] name = "nomos-libp2p" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "backon", @@ -4684,7 +4687,7 @@ dependencies = [ [[package]] name = "nomos-network" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "cryptarchia-sync", @@ -4703,7 +4706,7 @@ dependencies = [ [[package]] name = "nomos-node" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "axum", @@ -4768,7 +4771,7 @@ dependencies = [ [[package]] name = "nomos-sdp" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "futures", @@ -4786,7 +4789,7 @@ dependencies = [ [[package]] name = "nomos-storage" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "bytes", @@ -4806,7 +4809,7 @@ dependencies = [ [[package]] name = "nomos-system-sig" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-ctrlc", "async-trait", @@ -4817,7 +4820,7 @@ dependencies = [ [[package]] name = "nomos-time" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "cfg_eval", @@ -4839,7 +4842,7 @@ dependencies = [ [[package]] name = "nomos-tracing" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "opentelemetry", "opentelemetry-http", @@ -4862,7 +4865,7 @@ dependencies = [ [[package]] name = "nomos-tracing-service" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "nomos-tracing", @@ -4876,7 +4879,7 @@ dependencies = [ [[package]] name = "nomos-utils" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "blake2", @@ -4893,7 +4896,7 @@ dependencies = [ [[package]] name = "nomos-wallet" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "bytes", @@ -5408,7 +5411,7 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "pol" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "circuits-prover", "circuits-utils", @@ -5438,7 +5441,7 @@ dependencies = [ [[package]] name = "poq" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "circuits-prover", "circuits-utils", @@ -5460,7 +5463,7 @@ checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "poseidon2" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "ark-bn254 0.4.0", "ark-ff 0.4.2", @@ -6482,7 +6485,7 @@ dependencies = [ [[package]] name = "services-utils" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "futures", @@ -6670,7 +6673,7 @@ dependencies = [ [[package]] name = "subnetworks-assignations" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "counter", "libp2p-identity", @@ -6970,7 +6973,7 @@ dependencies = [ [[package]] name = "tests" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "blst", @@ -7594,7 +7597,7 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tx-service" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "async-trait", "futures", @@ -7754,7 +7757,7 @@ checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" [[package]] name = "utxotree" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "ark-ff 0.4.2", "groth16", @@ -7807,7 +7810,7 @@ dependencies = [ [[package]] name = "wallet" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "key-management-system-keys", "nomos-core", @@ -8360,7 +8363,7 @@ checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "witness-generator" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "circuits-utils", "tempfile", @@ -8570,7 +8573,7 @@ dependencies = [ [[package]] name = "zksign" version = "0.1.0" -source = "git+https://github.com/logos-co/nomos-node.git?rev=3f623e0c9d159bd8b398a25eda8db0642c491f99#3f623e0c9d159bd8b398a25eda8db0642c491f99" +source = "git+https://github.com/logos-co/nomos-node.git?rev=1fce2dc3f482c16361316eb2a1b6ccd1206aa917#1fce2dc3f482c16361316eb2a1b6ccd1206aa917" dependencies = [ "circuits-prover", "circuits-utils", diff --git a/Cargo.toml b/Cargo.toml index d7a3054..fcd31af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,50 +38,50 @@ testing-framework-runner-local = { default-features = false, path = "testing-f testing-framework-workflows = { default-features = false, path = "testing-framework/workflows" } # Nomos git dependencies (pinned to latest master) -broadcast-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } +broadcast-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } cfgsync = { default-features = false, path = "testing-framework/tools/cfgsync" } -chain-leader = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99", features = [ +chain-leader = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917", features = [ "pol-dev-mode", ] } -chain-network = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -chain-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -common-http-client = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -cryptarchia-engine = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -cryptarchia-sync = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -executor-http-client = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -groth16 = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -key-management-system-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -kzgrs = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -kzgrs-backend = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-api = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-blend-message = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-blend-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-cli = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-core = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-da-dispersal = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-da-network-core = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-da-network-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-da-sampling = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-da-verifier = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-executor = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-http-api-common = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-ledger = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-libp2p = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-network = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-node = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-sdp = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-time = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-tracing = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-tracing-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-utils = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -nomos-wallet = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -poc = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -pol = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -subnetworks-assignations = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -tests = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -tx-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -wallet = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } -zksign = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "3f623e0c9d159bd8b398a25eda8db0642c491f99" } +chain-network = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +chain-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +common-http-client = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +cryptarchia-engine = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +cryptarchia-sync = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +executor-http-client = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +groth16 = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +key-management-system-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +kzgrs = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +kzgrs-backend = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-api = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-blend-message = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-blend-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-cli = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-core = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-da-dispersal = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-da-network-core = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-da-network-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-da-sampling = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-da-verifier = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-executor = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-http-api-common = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-ledger = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-libp2p = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-network = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-node = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-sdp = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-time = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-tracing = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-tracing-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-utils = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +nomos-wallet = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +poc = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +pol = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +subnetworks-assignations = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +tests = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +tx-service = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +wallet = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } +zksign = { default-features = false, git = "https://github.com/logos-co/nomos-node.git", rev = "1fce2dc3f482c16361316eb2a1b6ccd1206aa917" } # External crates async-trait = { default-features = false, version = "0.1" } diff --git a/examples/src/bin/compose_runner.rs b/examples/src/bin/compose_runner.rs index 47a1bae..e817e49 100644 --- a/examples/src/bin/compose_runner.rs +++ b/examples/src/bin/compose_runner.rs @@ -75,9 +75,7 @@ async fn run_compose_case( "building scenario plan" ); - let chaos_min_delay = Duration::from_secs(CHAOS_MIN_DELAY_SECS) - .max(run_duration + Duration::from_secs(CHAOS_DELAY_HEADROOM_SECS)); - let chaos_max_delay = Duration::from_secs(CHAOS_MAX_DELAY_SECS).max(chaos_min_delay); + let (chaos_min_delay, chaos_max_delay, chaos_target_cooldown) = chaos_timings(run_duration); let mut plan = ScenarioBuilder::topology_with(|t| { t.network_star().validators(validators).executors(executors) @@ -88,7 +86,7 @@ async fn run_compose_case( // Keep chaos restarts outside the test run window to avoid crash loops on restart. .min_delay(chaos_min_delay) .max_delay(chaos_max_delay) - .target_cooldown(Duration::from_secs(CHAOS_COOLDOWN_SECS)) + .target_cooldown(chaos_target_cooldown) .apply() }) .wallets(TOTAL_WALLETS) @@ -121,3 +119,23 @@ async fn run_compose_case( .context("running compose scenario failed")?; Ok(()) } + +fn chaos_timings(run_duration: Duration) -> (Duration, Duration, Duration) { + let headroom = Duration::from_secs(CHAOS_DELAY_HEADROOM_SECS); + let chaos_min_delay = Duration::from_secs(CHAOS_MIN_DELAY_SECS).max(run_duration + headroom); + let chaos_max_delay = Duration::from_secs(CHAOS_MAX_DELAY_SECS).max(chaos_min_delay); + let chaos_target_cooldown = Duration::from_secs(CHAOS_COOLDOWN_SECS).max(chaos_min_delay); + + (chaos_min_delay, chaos_max_delay, chaos_target_cooldown) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chaos_cooldown_is_never_less_than_min_delay() { + let (min_delay, _max_delay, cooldown) = chaos_timings(Duration::from_secs(600)); + assert!(cooldown >= min_delay); + } +} diff --git a/scripts/build-bundle.sh b/scripts/build-bundle.sh index e1d5aaa..3b5f397 100755 --- a/scripts/build-bundle.sh +++ b/scripts/build-bundle.sh @@ -1,294 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -# Build a nomos-binaries.tar.gz for the specified platform. -# -# Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH] [--rev REV | --path DIR] [--features LIST] [--docker-platform PLATFORM] -# --platform Target platform for binaries (default: host) -# --output Output path for the tarball (default: .tmp/nomos-binaries--.tar.gz) -# --rev nomos-node git revision to build (overrides NOMOS_NODE_REV) -# --path Use local nomos-node checkout at DIR (skip fetch/checkout) -# --features Extra cargo features to enable (comma-separated); base always includes "testing" -# --docker-platform Docker platform for Linux bundle when running on non-Linux host (default: auto; linux/arm64 on Apple silicon Docker Desktop, else linux/amd64) - -# Always run under bash; bail out if someone invokes via sh. +# Thin wrapper; the actual implementation lives in scripts/lib/build-bundle.sh if [ -z "${BASH_VERSION:-}" ]; then exec bash "$0" "$@" fi -usage() { - cat <<'USAGE' -Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH] - -Options: - --platform Target platform for binaries (default: host) - --output Output path for the tarball (default: .tmp/nomos-binaries--.tar.gz) - --rev nomos-node git revision to build (overrides NOMOS_NODE_REV) - --path Use local nomos-node checkout at DIR (skip fetch/checkout) - --features Extra cargo features to enable (comma-separated); base always includes "testing" - --docker-platform Docker platform for Linux bundle when running on non-Linux host (default: auto; linux/arm64 on Apple silicon Docker Desktop, else linux/amd64) - -Notes: - - For compose/k8s, use platform=linux. If running on macOS, this script will - run inside a Linux Docker container to produce Linux binaries. - - On Apple silicon, Docker defaults to linux/arm64; for compose/k8s you likely - want linux/amd64 (the default here). Override with --docker-platform. - - VERSION, NOMOS_NODE_REV, and optional NOMOS_NODE_PATH env vars are honored (defaults align with run-examples.sh). -USAGE -} - -fail() { echo "$1" >&2; exit 1; } - -if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 -fi - -NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}" - ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [ -f "${ROOT_DIR}/versions.env" ]; then - # shellcheck disable=SC1091 - . "${ROOT_DIR}/versions.env" -else - echo "ERROR: versions.env missing; run from repo root or restore the file." >&2 - exit 1 -fi -DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}" -DEFAULT_NODE_REV="${NOMOS_NODE_REV:-}" -DEFAULT_NODE_PATH="${NOMOS_NODE_PATH:-}" -PLATFORM="host" -OUTPUT="" -REV_OVERRIDE="" -PATH_OVERRIDE="" -DOCKER_PLATFORM="${NOMOS_BUNDLE_DOCKER_PLATFORM:-${NOMOS_BIN_PLATFORM:-}}" -BUNDLE_RUSTUP_TOOLCHAIN="${BUNDLE_RUSTUP_TOOLCHAIN:-}" -if [ -z "${BUNDLE_RUSTUP_TOOLCHAIN}" ] && command -v rustup >/dev/null 2>&1 && [ -f "${ROOT_DIR}/rust-toolchain.toml" ]; then - BUNDLE_RUSTUP_TOOLCHAIN="$(awk -F '\"' '/^[[:space:]]*channel[[:space:]]*=/{print $2; exit}' "${ROOT_DIR}/rust-toolchain.toml")" -fi +# shellcheck disable=SC1091 +. "${ROOT_DIR}/scripts/lib/build-bundle.sh" -# Default Docker platform to the engine architecture when possible. -if [ -z "${DOCKER_PLATFORM}" ] && command -v docker >/dev/null 2>&1; then - docker_arch="$(docker version --format '{{.Server.Arch}}' 2>/dev/null || true)" - case "${docker_arch}" in - arm64|aarch64) DOCKER_PLATFORM="linux/arm64" ;; - amd64|x86_64) DOCKER_PLATFORM="linux/amd64" ;; - *) DOCKER_PLATFORM="linux/amd64" ;; - esac -fi -DOCKER_PLATFORM="${DOCKER_PLATFORM:-linux/amd64}" - -# To avoid confusing cache corruption errors inside the Dockerized Linux build, -# always start from a clean cargo registry/git cache for the cross-build. -rm -rf "${ROOT_DIR}/.tmp/cargo-linux/registry" "${ROOT_DIR}/.tmp/cargo-linux/git" - -while [ "$#" -gt 0 ]; do - case "$1" in - --platform=*|-p=*) PLATFORM="${1#*=}"; shift ;; - --platform|-p) PLATFORM="${2:-}"; shift 2 ;; - --output=*|-o=*) OUTPUT="${1#*=}"; shift ;; - --output|-o) OUTPUT="${2:-}"; shift 2 ;; - --rev=*) REV_OVERRIDE="${1#*=}"; shift ;; - --rev) REV_OVERRIDE="${2:-}"; shift 2 ;; - --path=*) PATH_OVERRIDE="${1#*=}"; shift ;; - --path) PATH_OVERRIDE="${2:-}"; shift 2 ;; - --features=*) NOMOS_EXTRA_FEATURES="${1#*=}"; shift ;; - --features) NOMOS_EXTRA_FEATURES="${2:-}"; shift 2 ;; - --docker-platform=*) DOCKER_PLATFORM="${1#*=}"; shift ;; - --docker-platform) DOCKER_PLATFORM="${2:-}"; shift 2 ;; - *) fail "Unknown argument: $1" ;; - esac -done - -case "$PLATFORM" in - host|linux) ;; - *) fail "--platform must be host or linux" ;; -esac - -VERSION="${DEFAULT_VERSION}" -if [ -n "${REV_OVERRIDE}" ] && [ -n "${PATH_OVERRIDE}" ]; then - fail "Use either --rev or --path, not both" -fi -if [ -z "${REV_OVERRIDE}" ] && [ -z "${PATH_OVERRIDE}" ] && [ -z "${DEFAULT_NODE_REV}" ] && [ -z "${DEFAULT_NODE_PATH}" ]; then - fail "Provide --rev, --path, or set NOMOS_NODE_REV/NOMOS_NODE_PATH in versions.env" -fi -NOMOS_NODE_REV="${REV_OVERRIDE:-${DEFAULT_NODE_REV}}" -NOMOS_NODE_PATH="${PATH_OVERRIDE:-${DEFAULT_NODE_PATH}}" - -# Normalize OUTPUT to an absolute path under the workspace. -if [ -z "${OUTPUT}" ]; then - OUTPUT="${ROOT_DIR}/.tmp/nomos-binaries-${PLATFORM}-${VERSION}.tar.gz" -elif [[ "${OUTPUT}" != /* ]]; then - OUTPUT="${ROOT_DIR}/${OUTPUT#./}" -fi -echo "Bundle output: ${OUTPUT}" - -if [ "$PLATFORM" = "linux" ] && [ "$(uname -s)" != "Linux" ] && [ -z "${BUNDLE_IN_CONTAINER:-}" ]; then - if ! command -v docker >/dev/null 2>&1; then - fail "Docker is required to build a Linux bundle from non-Linux host" - fi - if [ -z "${DOCKER_PLATFORM}" ]; then - fail "--docker-platform must not be empty" - fi - - NODE_PATH_ENV="${NOMOS_NODE_PATH}" - EXTRA_MOUNTS=() - if [ -n "${NOMOS_NODE_PATH}" ]; then - case "${NOMOS_NODE_PATH}" in - "${ROOT_DIR}"/*) - NODE_PATH_ENV="/workspace${NOMOS_NODE_PATH#"${ROOT_DIR}"}" - ;; - /*) - NODE_PATH_ENV="/external/nomos-node" - EXTRA_MOUNTS+=("-v" "${NOMOS_NODE_PATH}:${NODE_PATH_ENV}") - ;; - *) - fail "--path must be absolute when cross-building in Docker" - ;; - esac - fi - - echo "==> Building Linux bundle inside Docker" - container_output="/workspace${OUTPUT#"${ROOT_DIR}"}" - mkdir -p "${ROOT_DIR}/.tmp/cargo-linux" "${ROOT_DIR}/.tmp/nomos-node-linux-target" - - FEATURES_ARGS=() - if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then - FEATURES_ARGS+=(--features "${NOMOS_EXTRA_FEATURES}") - fi - - SRC_ARGS=() - if [ -n "${NODE_PATH_ENV}" ]; then - SRC_ARGS+=(--path "${NODE_PATH_ENV}") - else - SRC_ARGS+=(--rev "${NOMOS_NODE_REV}") - fi - - docker run --rm --platform "${DOCKER_PLATFORM}" \ - -e VERSION="$VERSION" \ - -e NOMOS_NODE_REV="$NOMOS_NODE_REV" \ - -e NOMOS_NODE_PATH="$NODE_PATH_ENV" \ - -e NOMOS_CIRCUITS="/workspace/.tmp/nomos-circuits-linux" \ - -e STACK_DIR="/workspace/.tmp/nomos-circuits-linux" \ - -e HOST_DIR="/workspace/.tmp/nomos-circuits-linux" \ - -e NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}" \ - -e BUNDLE_IN_CONTAINER=1 \ - -e CARGO_HOME=/workspace/.tmp/cargo-linux \ - -e CARGO_TARGET_DIR=/workspace/.tmp/nomos-node-linux-target \ - -v "${ROOT_DIR}/.tmp/cargo-linux":/workspace/.tmp/cargo-linux \ - -v "${ROOT_DIR}/.tmp/nomos-node-linux-target":/workspace/.tmp/nomos-node-linux-target \ - -v "$ROOT_DIR":/workspace \ - "${EXTRA_MOUNTS[@]}" \ - -w /workspace \ - rust:1.80-bullseye \ - bash -c "apt-get update && apt-get install -y clang llvm-dev libclang-dev pkg-config cmake libssl-dev rsync libgmp10 libgmp-dev libgomp1 nasm && ./scripts/build-bundle.sh --platform linux --output \"${container_output}\" ${SRC_ARGS[*]} ${FEATURES_ARGS[*]}" - - exit 0 -fi - -echo "==> Preparing circuits (version ${VERSION})" -if [ "$PLATFORM" = "host" ]; then - CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-host" - NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-host-target" -else - CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-linux" - NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-linux-target" -fi - -NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-${PLATFORM}-src" -NODE_SRC="${NOMOS_NODE_PATH:-${NODE_SRC_DEFAULT}}" -if [ -n "${NOMOS_NODE_PATH}" ]; then - if [ ! -d "${NODE_SRC}" ]; then - fail "NOMOS_NODE_PATH does not exist: ${NODE_SRC}" - fi - rm -rf "${NODE_SRC_DEFAULT}" - if [ -d "${NODE_TARGET}" ]; then - find "${NODE_TARGET}" -mindepth 1 -maxdepth 1 -exec rm -rf {} + - fi - NODE_TARGET="${NODE_TARGET}-local" -fi - -export NOMOS_CIRCUITS="${CIRCUITS_DIR}" -mkdir -p "${ROOT_DIR}/.tmp" "${CIRCUITS_DIR}" -if [ -f "${CIRCUITS_DIR}/${KZG_FILE:-kzgrs_test_params}" ]; then - echo "Circuits already present at ${CIRCUITS_DIR}; skipping download" -else - STACK_DIR="${CIRCUITS_DIR}" HOST_DIR="${CIRCUITS_DIR}" \ - "${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}" Building binaries (platform=${PLATFORM})" -mkdir -p "${NODE_SRC}" -( - cd "${NODE_SRC}" - if [ -n "${NOMOS_NODE_PATH}" ]; then - echo "Using local nomos-node checkout at ${NODE_SRC} (no fetch/checkout)" - else - if [ ! -d "${NODE_SRC}/.git" ]; then - git clone https://github.com/logos-co/nomos-node.git "${NODE_SRC}" - fi - git fetch --depth 1 origin "${NOMOS_NODE_REV}" - git checkout "${NOMOS_NODE_REV}" - git reset --hard - git clean -fdx - fi - if [ -n "${BUNDLE_RUSTUP_TOOLCHAIN}" ]; then - RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \ - RUSTUP_TOOLCHAIN="${BUNDLE_RUSTUP_TOOLCHAIN}" \ - cargo build --features "${FEATURES}" \ - -p nomos-node -p nomos-executor -p nomos-cli \ - --target-dir "${NODE_TARGET}" - else - RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \ - cargo build --features "${FEATURES}" \ - -p nomos-node -p nomos-executor -p nomos-cli \ - --target-dir "${NODE_TARGET}" - fi -) - -echo "==> Packaging bundle" -bundle_dir="${ROOT_DIR}/.tmp/nomos-bundle" -rm -rf "${bundle_dir}" -mkdir -p "${bundle_dir}/artifacts/circuits" -cp -a "${CIRCUITS_DIR}/." "${bundle_dir}/artifacts/circuits/" -mkdir -p "${bundle_dir}/artifacts" -cp "${NODE_BIN}" "${bundle_dir}/artifacts/" -cp "${EXEC_BIN}" "${bundle_dir}/artifacts/" -cp "${CLI_BIN}" "${bundle_dir}/artifacts/" -{ - echo "nomos_node_path=${NOMOS_NODE_PATH:-}" - echo "nomos_node_rev=${NOMOS_NODE_REV:-}" - if [ -d "${NODE_SRC}/.git" ] && command -v git >/dev/null 2>&1; then - echo "nomos_node_git_head=$(git -C "${NODE_SRC}" rev-parse HEAD 2>/dev/null || true)" - fi - echo "platform=${PLATFORM}" - echo "features=${FEATURES}" -} > "${bundle_dir}/artifacts/nomos-bundle-meta.env" - -mkdir -p "$(dirname "${OUTPUT}")" -if tar --help 2>/dev/null | grep -q -- '--no-mac-metadata'; then - tar --no-mac-metadata --no-xattrs -czf "${OUTPUT}" -C "${bundle_dir}" artifacts -elif tar --help 2>/dev/null | grep -q -- '--no-xattrs'; then - tar --no-xattrs -czf "${OUTPUT}" -C "${bundle_dir}" artifacts -else - tar -czf "${OUTPUT}" -C "${bundle_dir}" artifacts -fi -echo "Bundle created at ${OUTPUT}" - -if [[ "${FEATURES}" == *profiling* ]]; then - cat <<'EOF_PROF' -Profiling endpoints (enabled by --features profiling): - CPU pprof (SVG): curl "http://:8722/debug/pprof/profile?seconds=15&format=svg" -o profile.svg - CPU pprof (proto): go tool pprof -http=:8080 "http://:8722/debug/pprof/profile?seconds=15&format=proto" -EOF_PROF -fi +build_bundle::main "$@" diff --git a/scripts/build-linux-binaries.sh b/scripts/build-linux-binaries.sh index c91030d..46e59c0 100755 --- a/scripts/build-linux-binaries.sh +++ b/scripts/build-linux-binaries.sh @@ -1,74 +1,14 @@ #!/usr/bin/env bash -# Build Linux nomos-node/nomos-executor/nomos-cli binaries and stage them into -# testing-framework/assets/stack/bin along with the circuits bundle. This uses -# a Dockerized toolchain so it can be run from macOS as well. set -euo pipefail -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [ -f "${ROOT_DIR}/paths.env" ]; then - # shellcheck disable=SC1091 - . "${ROOT_DIR}/paths.env" +# Thin wrapper; the actual implementation lives in scripts/lib/build-linux-binaries.sh +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" fi -NOMOS_NODE_REV="${NOMOS_NODE_REV:-d2dd5a5084e1daef4032562c77d41de5e4d495f8}" -NOMOS_CIRCUITS_VERSION="${NOMOS_CIRCUITS_VERSION:-v0.3.1}" -NOMOS_BIN_PLATFORM="${NOMOS_BIN_PLATFORM:-linux/amd64}" -case "${NOMOS_BIN_PLATFORM}" in - linux/amd64) CIRCUITS_PLATFORM="linux-x86_64" ;; - linux/arm64) CIRCUITS_PLATFORM="linux-aarch64" ;; - *) echo "Unsupported platform ${NOMOS_BIN_PLATFORM}. Use linux/amd64 or linux/arm64." >&2; exit 1 ;; -esac +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -echo "Workspace: ${ROOT_DIR}" -echo "Nomos node rev: ${NOMOS_NODE_REV}" -echo "Circuits version: ${NOMOS_CIRCUITS_VERSION}" +# shellcheck disable=SC1091 +. "${ROOT_DIR}/scripts/lib/build-linux-binaries.sh" -BIN_OUT="${ROOT_DIR}/testing-framework/assets/stack/bin" -KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}" -CIRCUITS_OUT="${ROOT_DIR}/${KZG_DIR_REL}" -SRC_DIR="${ROOT_DIR}/.tmp/nomos-node-src" -CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits" - -rm -rf "${CIRCUITS_OUT}" -mkdir -p "${BIN_OUT}" "${CIRCUITS_OUT}" "${SRC_DIR}" "${CIRCUITS_DIR}" - -docker run --rm --platform "${NOMOS_BIN_PLATFORM}" \ - -v "${ROOT_DIR}:/workspace" \ - -w /workspace \ - -e NOMOS_NODE_REV="${NOMOS_NODE_REV}" \ - -e NOMOS_CIRCUITS_VERSION="${NOMOS_CIRCUITS_VERSION}" \ - -e NOMOS_CIRCUITS_PLATFORM="${CIRCUITS_PLATFORM}" \ - rust:1.91.0-slim-bookworm \ - bash -euo pipefail -c ' - apt-get update && apt-get install -y git clang llvm-dev libclang-dev pkg-config cmake libssl-dev rsync libgmp-dev libgomp1 nasm curl ca-certificates xz-utils - cd /workspace - RAPIDSNARK_BUILD_GMP=0 RAPIDSNARK_USE_ASM=OFF \ - ./scripts/setup-nomos-circuits.sh "${NOMOS_CIRCUITS_VERSION}" "/workspace/.tmp/nomos-circuits" - - if [ ! -d /workspace/.tmp/nomos-node-src/.git ]; then - git clone https://github.com/logos-co/nomos-node.git /workspace/.tmp/nomos-node-src - fi - cd /workspace/.tmp/nomos-node-src - git fetch --depth 1 origin "${NOMOS_NODE_REV}" - git checkout "${NOMOS_NODE_REV}" - git reset --hard - git clean -fdx - - NOMOS_CIRCUITS=/workspace/.tmp/nomos-circuits \ - cargo build --features "testing" \ - -p nomos-node -p nomos-executor -p nomos-cli - - cp /workspace/.tmp/nomos-node-src/target/debug/nomos-node /workspace/testing-framework/assets/stack/bin/ - cp /workspace/.tmp/nomos-node-src/target/debug/nomos-executor /workspace/testing-framework/assets/stack/bin/ - cp /workspace/.tmp/nomos-node-src/target/debug/nomos-cli /workspace/testing-framework/assets/stack/bin/ - rsync -a /workspace/.tmp/nomos-circuits/ /workspace/testing-framework/assets/stack/kzgrs_test_params/ - ' - -# Ensure host ownership of staged artifacts. -chown -R "$(id -u)":"$(id -g)" "${BIN_OUT}" "${CIRCUITS_OUT}" "${SRC_DIR}" "${CIRCUITS_DIR}" 2>/dev/null || true - -echo -echo "Binaries staged in ${BIN_OUT}:" -ls -l "${BIN_OUT}" -echo -echo "Circuits staged in ${CIRCUITS_OUT}" +build_linux_binaries::main "$@" diff --git a/scripts/lib/build-bundle.sh b/scripts/lib/build-bundle.sh new file mode 100644 index 0000000..b0f4c59 --- /dev/null +++ b/scripts/lib/build-bundle.sh @@ -0,0 +1,369 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Intended to be sourced by scripts/build-bundle.sh +# shellcheck disable=SC1091 +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh" + +build_bundle::usage() { + cat <<'USAGE' +Usage: scripts/build-bundle.sh [--platform host|linux] [--output PATH] + +Options: + --platform Target platform for binaries (default: host) + --output Output path for the tarball (default: .tmp/nomos-binaries--.tar.gz) + --rev nomos-node git revision to build (overrides NOMOS_NODE_REV) + --path Use local nomos-node checkout at DIR (skip fetch/checkout) + --features Extra cargo features to enable (comma-separated); base always includes "testing" + --docker-platform Docker platform for Linux bundle when running on non-Linux host (default: auto; linux/arm64 on Apple silicon Docker Desktop, else linux/amd64) + +Notes: + - For compose/k8s, use platform=linux. If running on macOS, this script will + run inside a Linux Docker container to produce Linux binaries. + - On Apple silicon, Docker defaults to linux/arm64; for compose/k8s you likely + want linux/amd64 (the default here). Override with --docker-platform. + - VERSION, NOMOS_NODE_REV, and optional NOMOS_NODE_PATH env vars are honored (defaults align with run-examples.sh). +USAGE +} + +build_bundle::fail() { + echo "$1" >&2 + exit 1 +} + +build_bundle::apply_nomos_node_patches() { + local node_src="$1" + + local apply="${NOMOS_NODE_APPLY_PATCHES:-1}" + if [ "${apply}" = "0" ]; then + return 0 + fi + + local patch_dir="${NOMOS_NODE_PATCH_DIR:-${ROOT_DIR}/patches/nomos-node}" + if [ ! -d "${patch_dir}" ]; then + return 0 + fi + + local level="${NOMOS_NODE_PATCH_LEVEL:-}" + if [ -z "${level}" ]; then + level="all" + fi + + shopt -s nullglob + local -a patches=("${patch_dir}"/*.patch) + shopt -u nullglob + if [ "${#patches[@]}" -eq 0 ]; then + return 0 + fi + + echo "==> Applying nomos-node patches from ${patch_dir} (level=${level})" + local patch base phase + for patch in "${patches[@]}"; do + base="$(basename "${patch}")" + phase="" + if [[ "${base}" =~ phase([0-9]+) ]]; then + phase="${BASH_REMATCH[1]}" + fi + if [ "${level}" != "all" ] && [ "${level}" != "ALL" ]; then + if ! [[ "${level}" =~ ^[0-9]+$ ]]; then + build_bundle::fail "Invalid NOMOS_NODE_PATCH_LEVEL: ${level} (expected integer or 'all')" + fi + if [ -n "${phase}" ] && [ "${phase}" -gt "${level}" ]; then + continue + fi + fi + + git -C "${node_src}" apply --whitespace=nowarn "${patch}" + done +} + +build_bundle::load_env() { + ROOT_DIR="$(common::repo_root)" + export ROOT_DIR + + common::require_file "${ROOT_DIR}/versions.env" + # shellcheck disable=SC1091 + . "${ROOT_DIR}/versions.env" + + DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}" + DEFAULT_NODE_REV="${NOMOS_NODE_REV:-}" + DEFAULT_NODE_PATH="${NOMOS_NODE_PATH:-}" + + NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}" + DOCKER_PLATFORM="${NOMOS_BUNDLE_DOCKER_PLATFORM:-${NOMOS_BIN_PLATFORM:-}}" + BUNDLE_RUSTUP_TOOLCHAIN="${BUNDLE_RUSTUP_TOOLCHAIN:-}" + + if [ -z "${BUNDLE_RUSTUP_TOOLCHAIN}" ] && command -v rustup >/dev/null 2>&1 && [ -f "${ROOT_DIR}/rust-toolchain.toml" ]; then + BUNDLE_RUSTUP_TOOLCHAIN="$(awk -F '\"' '/^[[:space:]]*channel[[:space:]]*=/{print $2; exit}' "${ROOT_DIR}/rust-toolchain.toml")" + fi +} + +build_bundle::default_docker_platform() { + if [ -n "${DOCKER_PLATFORM}" ]; then + return 0 + fi + if ! command -v docker >/dev/null 2>&1; then + return 0 + fi + local docker_arch + docker_arch="$(docker version --format '{{.Server.Arch}}' 2>/dev/null || true)" + case "${docker_arch}" in + arm64|aarch64) DOCKER_PLATFORM="linux/arm64" ;; + amd64|x86_64) DOCKER_PLATFORM="linux/amd64" ;; + *) DOCKER_PLATFORM="linux/amd64" ;; + esac +} + +build_bundle::parse_args() { + PLATFORM="host" + OUTPUT="" + REV_OVERRIDE="" + PATH_OVERRIDE="" + + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + build_bundle::usage + exit 0 + fi + + while [ "$#" -gt 0 ]; do + case "$1" in + --platform=*|-p=*) PLATFORM="${1#*=}"; shift ;; + --platform|-p) PLATFORM="${2:-}"; shift 2 ;; + --output=*|-o=*) OUTPUT="${1#*=}"; shift ;; + --output|-o) OUTPUT="${2:-}"; shift 2 ;; + --rev=*) REV_OVERRIDE="${1#*=}"; shift ;; + --rev) REV_OVERRIDE="${2:-}"; shift 2 ;; + --path=*) PATH_OVERRIDE="${1#*=}"; shift ;; + --path) PATH_OVERRIDE="${2:-}"; shift 2 ;; + --features=*) NOMOS_EXTRA_FEATURES="${1#*=}"; shift ;; + --features) NOMOS_EXTRA_FEATURES="${2:-}"; shift 2 ;; + --docker-platform=*) DOCKER_PLATFORM="${1#*=}"; shift ;; + --docker-platform) DOCKER_PLATFORM="${2:-}"; shift 2 ;; + *) build_bundle::fail "Unknown argument: $1" ;; + esac + done +} + +build_bundle::validate_and_finalize() { + case "${PLATFORM}" in + host|linux) ;; + *) build_bundle::fail "--platform must be host or linux" ;; + esac + + VERSION="${DEFAULT_VERSION}" + + if [ -n "${REV_OVERRIDE}" ] && [ -n "${PATH_OVERRIDE}" ]; then + build_bundle::fail "Use either --rev or --path, not both" + fi + if [ -z "${REV_OVERRIDE}" ] && [ -z "${PATH_OVERRIDE}" ] && [ -z "${DEFAULT_NODE_REV}" ] && [ -z "${DEFAULT_NODE_PATH}" ]; then + build_bundle::fail "Provide --rev, --path, or set NOMOS_NODE_REV/NOMOS_NODE_PATH in versions.env" + fi + NOMOS_NODE_REV="${REV_OVERRIDE:-${DEFAULT_NODE_REV}}" + NOMOS_NODE_PATH="${PATH_OVERRIDE:-${DEFAULT_NODE_PATH}}" + export NOMOS_NODE_REV NOMOS_NODE_PATH + + build_bundle::default_docker_platform + DOCKER_PLATFORM="${DOCKER_PLATFORM:-linux/amd64}" + + # Normalize OUTPUT to an absolute path under the workspace. + if [ -z "${OUTPUT}" ]; then + OUTPUT="${ROOT_DIR}/.tmp/nomos-binaries-${PLATFORM}-${VERSION}.tar.gz" + elif [[ "${OUTPUT}" != /* ]]; then + OUTPUT="${ROOT_DIR}/${OUTPUT#./}" + fi + echo "Bundle output: ${OUTPUT}" +} + +build_bundle::clean_cargo_linux_cache() { + rm -rf "${ROOT_DIR}/.tmp/cargo-linux/registry" "${ROOT_DIR}/.tmp/cargo-linux/git" +} + +build_bundle::maybe_run_linux_build_in_docker() { + # With `set -e`, this function must return 0 when no Docker cross-build is needed. + if [ "${PLATFORM}" != "linux" ] || [ "$(uname -s)" = "Linux" ] || [ -n "${BUNDLE_IN_CONTAINER:-}" ]; then + return 0 + fi + + command -v docker >/dev/null 2>&1 || build_bundle::fail "Docker is required to build a Linux bundle from non-Linux host" + [ -n "${DOCKER_PLATFORM}" ] || build_bundle::fail "--docker-platform must not be empty" + + local node_path_env="${NOMOS_NODE_PATH}" + local -a extra_mounts=() + if [ -n "${NOMOS_NODE_PATH}" ]; then + case "${NOMOS_NODE_PATH}" in + "${ROOT_DIR}"/*) + node_path_env="/workspace${NOMOS_NODE_PATH#"${ROOT_DIR}"}" + ;; + /*) + node_path_env="/external/nomos-node" + extra_mounts+=("-v" "${NOMOS_NODE_PATH}:${node_path_env}") + ;; + *) + build_bundle::fail "--path must be absolute when cross-building in Docker" + ;; + esac + fi + + echo "==> Building Linux bundle inside Docker" + local container_output="/workspace${OUTPUT#"${ROOT_DIR}"}" + mkdir -p "${ROOT_DIR}/.tmp/cargo-linux" "${ROOT_DIR}/.tmp/nomos-node-linux-target" + + local -a features_args=() + if [ -n "${NOMOS_EXTRA_FEATURES:-}" ]; then + features_args+=(--features "${NOMOS_EXTRA_FEATURES}") + fi + + local -a src_args=() + if [ -n "${node_path_env}" ]; then + src_args+=(--path "${node_path_env}") + else + src_args+=(--rev "${NOMOS_NODE_REV}") + fi + + docker run --rm --platform "${DOCKER_PLATFORM}" \ + -e VERSION="${VERSION}" \ + -e NOMOS_NODE_REV="${NOMOS_NODE_REV}" \ + -e NOMOS_NODE_PATH="${node_path_env}" \ + -e NOMOS_CIRCUITS="/workspace/.tmp/nomos-circuits-linux" \ + -e STACK_DIR="/workspace/.tmp/nomos-circuits-linux" \ + -e HOST_DIR="/workspace/.tmp/nomos-circuits-linux" \ + -e NOMOS_EXTRA_FEATURES="${NOMOS_EXTRA_FEATURES:-}" \ + -e BUNDLE_IN_CONTAINER=1 \ + -e CARGO_HOME=/workspace/.tmp/cargo-linux \ + -e CARGO_TARGET_DIR=/workspace/.tmp/nomos-node-linux-target \ + -v "${ROOT_DIR}/.tmp/cargo-linux":/workspace/.tmp/cargo-linux \ + -v "${ROOT_DIR}/.tmp/nomos-node-linux-target":/workspace/.tmp/nomos-node-linux-target \ + -v "${ROOT_DIR}:/workspace" \ + "${extra_mounts[@]}" \ + -w /workspace \ + rust:1.80-bullseye \ + bash -c "apt-get update && apt-get install -y clang llvm-dev libclang-dev pkg-config cmake libssl-dev rsync libgmp10 libgmp-dev libgomp1 nasm && ./scripts/build-bundle.sh --platform linux --output \"${container_output}\" ${src_args[*]} ${features_args[*]}" + + exit 0 +} + +build_bundle::prepare_circuits() { + echo "==> Preparing circuits (version ${VERSION})" + if [ "${PLATFORM}" = "host" ]; then + CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-host" + NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-host-target" + else + CIRCUITS_DIR="${ROOT_DIR}/.tmp/nomos-circuits-linux" + NODE_TARGET="${ROOT_DIR}/.tmp/nomos-node-linux-target" + fi + + NODE_SRC_DEFAULT="${ROOT_DIR}/.tmp/nomos-node-${PLATFORM}-src" + NODE_SRC="${NOMOS_NODE_PATH:-${NODE_SRC_DEFAULT}}" + if [ -n "${NOMOS_NODE_PATH}" ]; then + [ -d "${NODE_SRC}" ] || build_bundle::fail "NOMOS_NODE_PATH does not exist: ${NODE_SRC}" + rm -rf "${NODE_SRC_DEFAULT}" + if [ -d "${NODE_TARGET}" ]; then + find "${NODE_TARGET}" -mindepth 1 -maxdepth 1 -exec rm -rf {} + + fi + NODE_TARGET="${NODE_TARGET}-local" + fi + + export NOMOS_CIRCUITS="${CIRCUITS_DIR}" + mkdir -p "${ROOT_DIR}/.tmp" "${CIRCUITS_DIR}" + if [ -f "${CIRCUITS_DIR}/${KZG_FILE:-kzgrs_test_params}" ]; then + echo "Circuits already present at ${CIRCUITS_DIR}; skipping download" + else + STACK_DIR="${CIRCUITS_DIR}" HOST_DIR="${CIRCUITS_DIR}" \ + "${ROOT_DIR}/scripts/setup-circuits-stack.sh" "${VERSION}" Building binaries (platform=${PLATFORM})" + mkdir -p "${NODE_SRC}" + ( + cd "${NODE_SRC}" + if [ -n "${NOMOS_NODE_PATH}" ]; then + echo "Using local nomos-node checkout at ${NODE_SRC} (no fetch/checkout)" + else + if [ ! -d "${NODE_SRC}/.git" ]; then + git clone https://github.com/logos-co/nomos-node.git "${NODE_SRC}" + fi + git fetch --depth 1 origin "${NOMOS_NODE_REV}" + git checkout "${NOMOS_NODE_REV}" + git reset --hard + git clean -fdx + fi + + if [ -z "${NOMOS_NODE_PATH}" ]; then + build_bundle::apply_nomos_node_patches "${NODE_SRC}" + fi + + if [ -n "${BUNDLE_RUSTUP_TOOLCHAIN}" ]; then + RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \ + RUSTUP_TOOLCHAIN="${BUNDLE_RUSTUP_TOOLCHAIN}" \ + cargo build --features "${FEATURES}" \ + -p nomos-node -p nomos-executor -p nomos-cli \ + --target-dir "${NODE_TARGET}" + else + RUSTFLAGS='--cfg feature="pol-dev-mode"' NOMOS_CIRCUITS="${CIRCUITS_DIR}" \ + cargo build --features "${FEATURES}" \ + -p nomos-node -p nomos-executor -p nomos-cli \ + --target-dir "${NODE_TARGET}" + fi + ) +} + +build_bundle::package_bundle() { + echo "==> Packaging bundle" + local bundle_dir="${ROOT_DIR}/.tmp/nomos-bundle" + rm -rf "${bundle_dir}" + mkdir -p "${bundle_dir}/artifacts/circuits" + cp -a "${CIRCUITS_DIR}/." "${bundle_dir}/artifacts/circuits/" + mkdir -p "${bundle_dir}/artifacts" + cp "${NODE_BIN}" "${bundle_dir}/artifacts/" + cp "${EXEC_BIN}" "${bundle_dir}/artifacts/" + cp "${CLI_BIN}" "${bundle_dir}/artifacts/" + { + echo "nomos_node_path=${NOMOS_NODE_PATH:-}" + echo "nomos_node_rev=${NOMOS_NODE_REV:-}" + if [ -d "${NODE_SRC}/.git" ] && command -v git >/dev/null 2>&1; then + echo "nomos_node_git_head=$(git -C "${NODE_SRC}" rev-parse HEAD 2>/dev/null || true)" + fi + echo "platform=${PLATFORM}" + echo "features=${FEATURES}" + } > "${bundle_dir}/artifacts/nomos-bundle-meta.env" + + mkdir -p "$(dirname "${OUTPUT}")" + if tar --help 2>/dev/null | grep -q -- '--no-mac-metadata'; then + tar --no-mac-metadata --no-xattrs -czf "${OUTPUT}" -C "${bundle_dir}" artifacts + elif tar --help 2>/dev/null | grep -q -- '--no-xattrs'; then + tar --no-xattrs -czf "${OUTPUT}" -C "${bundle_dir}" artifacts + else + tar -czf "${OUTPUT}" -C "${bundle_dir}" artifacts + fi + echo "Bundle created at ${OUTPUT}" + + if [[ "${FEATURES}" == *profiling* ]]; then + cat <<'EOF_PROF' +Profiling endpoints (enabled by --features profiling): + CPU pprof (SVG): curl "http://:8722/debug/pprof/profile?seconds=15&format=svg" -o profile.svg + CPU pprof (proto): go tool pprof -http=:8080 "http://:8722/debug/pprof/profile?seconds=15&format=proto" +EOF_PROF + fi +} + +build_bundle::main() { + build_bundle::load_env + build_bundle::clean_cargo_linux_cache + build_bundle::parse_args "$@" + build_bundle::validate_and_finalize + build_bundle::maybe_run_linux_build_in_docker + build_bundle::prepare_circuits + build_bundle::build_binaries + build_bundle::package_bundle +} diff --git a/scripts/lib/build-linux-binaries.sh b/scripts/lib/build-linux-binaries.sh new file mode 100644 index 0000000..6fba519 --- /dev/null +++ b/scripts/lib/build-linux-binaries.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Intended to be sourced by scripts/build-linux-binaries.sh +# shellcheck disable=SC1091 +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh" + +build_linux_binaries::usage() { + cat <<'EOF' +Usage: scripts/build-linux-binaries.sh [options] + +Builds a Linux bundle via scripts/build-bundle.sh, then stages artifacts into: + - testing-framework/assets/stack/bin + - testing-framework/assets/stack/kzgrs_test_params (or NOMOS_KZG_DIR_REL) + +Options: + --rev REV nomos-node git revision to build (overrides NOMOS_NODE_REV) + --path DIR use local nomos-node checkout (skip fetch/checkout) + --features LIST extra cargo features (comma-separated); base includes "testing" + --docker-platform PLAT docker platform for the Linux build (e.g. linux/amd64, linux/arm64) + --tar PATH stage from an existing bundle tarball (skip build) + --output PATH where to write the bundle tarball when building (default: .tmp/nomos-binaries-linux-.tar.gz) + -h, --help show help + +Environment: + VERSION circuits version (default from versions.env) + NOMOS_CIRCUITS_VERSION legacy alias for VERSION (supported) + NOMOS_NODE_REV default nomos-node revision (from versions.env) + NOMOS_KZG_DIR_REL host path for staged circuits dir (default: testing-framework/assets/stack/kzgrs_test_params) +EOF +} + +build_linux_binaries::fail_with_usage() { + echo "$1" >&2 + build_linux_binaries::usage + exit 1 +} + +build_linux_binaries::load_env() { + ROOT_DIR="$(common::repo_root)" + export ROOT_DIR + + common::require_file "${ROOT_DIR}/versions.env" + # shellcheck disable=SC1091 + . "${ROOT_DIR}/versions.env" + common::maybe_source "${ROOT_DIR}/paths.env" + + DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}" + VERSION="${VERSION:-${DEFAULT_VERSION}}" + if [ -n "${NOMOS_CIRCUITS_VERSION:-}" ]; then + VERSION="${NOMOS_CIRCUITS_VERSION}" + fi +} + +build_linux_binaries::parse_args() { + REV_OVERRIDE="" + PATH_OVERRIDE="" + EXTRA_FEATURES="" + DOCKER_PLATFORM="" + OUTPUT_TAR="" + INPUT_TAR="" + + while [ "$#" -gt 0 ]; do + case "$1" in + -h|--help) build_linux_binaries::usage; exit 0 ;; + --rev) REV_OVERRIDE="${2:-}"; shift 2 ;; + --rev=*) REV_OVERRIDE="${1#*=}"; shift ;; + --path) PATH_OVERRIDE="${2:-}"; shift 2 ;; + --path=*) PATH_OVERRIDE="${1#*=}"; shift ;; + --features) EXTRA_FEATURES="${2:-}"; shift 2 ;; + --features=*) EXTRA_FEATURES="${1#*=}"; shift ;; + --docker-platform) DOCKER_PLATFORM="${2:-}"; shift 2 ;; + --docker-platform=*) DOCKER_PLATFORM="${1#*=}"; shift ;; + --tar) INPUT_TAR="${2:-}"; shift 2 ;; + --tar=*) INPUT_TAR="${1#*=}"; shift ;; + --output|-o) OUTPUT_TAR="${2:-}"; shift 2 ;; + --output=*|-o=*) OUTPUT_TAR="${1#*=}"; shift ;; + *) build_linux_binaries::fail_with_usage "Unknown argument: $1" ;; + esac + done + + if [ -n "${REV_OVERRIDE}" ] && [ -n "${PATH_OVERRIDE}" ]; then + build_linux_binaries::fail_with_usage "Use either --rev or --path, not both" + fi + if [ -n "${INPUT_TAR}" ] && [ ! -f "${INPUT_TAR}" ]; then + build_linux_binaries::fail_with_usage "Bundle tarball not found: ${INPUT_TAR}" + fi + + if [ -z "${OUTPUT_TAR}" ]; then + OUTPUT_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz" + elif [[ "${OUTPUT_TAR}" != /* ]]; then + OUTPUT_TAR="${ROOT_DIR}/${OUTPUT_TAR#./}" + fi +} + +build_linux_binaries::build_bundle_if_needed() { + if [ -n "${INPUT_TAR}" ]; then + BUNDLE_TAR="${INPUT_TAR}" + return 0 + fi + + mkdir -p "$(dirname "${OUTPUT_TAR}")" + BUILD_ARGS=(--platform linux --output "${OUTPUT_TAR}") + if [ -n "${REV_OVERRIDE}" ]; then + BUILD_ARGS+=(--rev "${REV_OVERRIDE}") + elif [ -n "${PATH_OVERRIDE}" ]; then + BUILD_ARGS+=(--path "${PATH_OVERRIDE}") + fi + if [ -n "${EXTRA_FEATURES}" ]; then + BUILD_ARGS+=(--features "${EXTRA_FEATURES}") + fi + if [ -n "${DOCKER_PLATFORM}" ]; then + BUILD_ARGS+=(--docker-platform "${DOCKER_PLATFORM}") + fi + + echo "==> Building Linux bundle" + VERSION="${VERSION}" "${ROOT_DIR}/scripts/build-bundle.sh" "${BUILD_ARGS[@]}" + + BUNDLE_TAR="${OUTPUT_TAR}" +} + +build_linux_binaries::stage_from_bundle() { + local tar_path="$1" + local extract_dir + extract_dir="$(common::tmpdir nomos-linux-bundle.XXXXXX)" + cleanup() { rm -rf "${extract_dir}" 2>/dev/null || true; } + trap cleanup EXIT + + echo "==> Extracting ${tar_path}" + tar -xzf "${tar_path}" -C "${extract_dir}" + + local artifacts="${extract_dir}/artifacts" + [ -f "${artifacts}/nomos-node" ] || common::die "Missing nomos-node in bundle: ${tar_path}" + [ -f "${artifacts}/nomos-executor" ] || common::die "Missing nomos-executor in bundle: ${tar_path}" + [ -f "${artifacts}/nomos-cli" ] || common::die "Missing nomos-cli in bundle: ${tar_path}" + [ -d "${artifacts}/circuits" ] || common::die "Missing circuits/ in bundle: ${tar_path}" + + local bin_out="${ROOT_DIR}/testing-framework/assets/stack/bin" + local kzg_dir_rel="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}" + local circuits_out="${ROOT_DIR}/${kzg_dir_rel}" + + echo "==> Staging binaries to ${bin_out}" + mkdir -p "${bin_out}" + cp "${artifacts}/nomos-node" "${artifacts}/nomos-executor" "${artifacts}/nomos-cli" "${bin_out}/" + + echo "==> Staging circuits to ${circuits_out}" + rm -rf "${circuits_out}" + mkdir -p "${circuits_out}" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete "${artifacts}/circuits/" "${circuits_out}/" + else + cp -a "${artifacts}/circuits/." "${circuits_out}/" + fi + + # If the tarball was produced inside Docker, it might be root-owned on the host. + chown -R "$(id -u)":"$(id -g)" "${bin_out}" "${circuits_out}" 2>/dev/null || true +} + +build_linux_binaries::main() { + build_linux_binaries::load_env + build_linux_binaries::parse_args "$@" + build_linux_binaries::build_bundle_if_needed + build_linux_binaries::stage_from_bundle "${BUNDLE_TAR}" + + echo + echo "Binaries staged in ${ROOT_DIR}/testing-framework/assets/stack/bin" + echo "Circuits staged in ${ROOT_DIR}/${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}" + echo "Bundle tarball: ${BUNDLE_TAR}" +} + diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh new file mode 100644 index 0000000..ec769a3 --- /dev/null +++ b/scripts/lib/common.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +common::ensure_bash() { + if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" + fi +} + +common::repo_root() { + local caller_source="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}" + local dir + dir="$(cd "$(dirname "${caller_source}")" && pwd)" + while true; do + if [ -f "${dir}/versions.env" ]; then + echo "${dir}" + return 0 + fi + if [ "${dir}" = "/" ]; then + common::die "Could not locate repo root (versions.env) from ${caller_source}" + fi + dir="$(cd "${dir}/.." && pwd)" + done +} + +common::die() { + echo "ERROR: $1" >&2 + exit 1 +} + +common::is_uint() { + [[ "${1:-}" =~ ^[0-9]+$ ]] +} + +common::require_file() { + local path="$1" + [ -f "${path}" ] || common::die "Missing required file: ${path}" +} + +common::maybe_source() { + local path="$1" + if [ -f "${path}" ]; then + # shellcheck disable=SC1090 + . "${path}" + fi +} + +common::tmpfile() { + # macOS and GNU mktemp have slightly different flags; -t works on macOS. + mktemp -t "${1:-tmp.XXXXXX}" +} + +common::tmpdir() { + # macOS and GNU mktemp have slightly different flags; -t works on macOS. + mktemp -d -t "${1:-tmpdir.XXXXXX}" +} diff --git a/scripts/lib/run-examples.sh b/scripts/lib/run-examples.sh new file mode 100644 index 0000000..0207a96 --- /dev/null +++ b/scripts/lib/run-examples.sh @@ -0,0 +1,517 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This file is meant to be sourced by scripts/run-examples.sh (and possibly others). +# shellcheck disable=SC1091 +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh" + +run_examples::usage() { + cat <<'EOF' +Usage: scripts/run-examples.sh [options] [compose|host|k8s] + +Modes: + compose Run examples/src/bin/compose_runner.rs (default) + host Run examples/src/bin/local_runner.rs + k8s Run examples/src/bin/k8s_runner.rs + +Options: + -t, --run-seconds N Duration to run the demo (required) + -v, --validators N Number of validators (required) + -e, --executors N Number of executors (required) + --bundle PATH Convenience alias for setting NOMOS_BINARIES_TAR=PATH + --local Use a local Docker image tag (default for docker-desktop k8s) + --ecr Use an ECR image reference (default for non-docker-desktop k8s) + --no-image-build Skip rebuilding the compose/k8s image (sets NOMOS_SKIP_IMAGE_BUILD=1) + +Environment: + VERSION Circuits version (default from versions.env) + NOMOS_TESTNET_IMAGE Image reference (overridden by --local/--ecr selection) + ECR_IMAGE Full image reference for --ecr (overrides ECR_REGISTRY/ECR_REPO/TAG) + ECR_REGISTRY Registry hostname for --ecr (default public.ecr.aws/r4s5t9y4) + ECR_REPO Repository path for --ecr (default logos/logos-blockchain) + TAG Tag for --ecr (default test) + NOMOS_TESTNET_IMAGE_PULL_POLICY K8s imagePullPolicy (default IfNotPresent; set to Always for --ecr) + NOMOS_BINARIES_TAR Path to prebuilt binaries/circuits tarball (default .tmp/nomos-binaries--.tar.gz) + NOMOS_SKIP_IMAGE_BUILD Set to 1 to skip rebuilding the compose/k8s image +EOF +} + +run_examples::fail_with_usage() { + echo "$1" >&2 + run_examples::usage + exit 1 +} + +run_examples::load_env() { + ROOT_DIR="$(common::repo_root)" + export ROOT_DIR + + common::require_file "${ROOT_DIR}/versions.env" + # shellcheck disable=SC1091 + . "${ROOT_DIR}/versions.env" + common::maybe_source "${ROOT_DIR}/paths.env" + + DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}" + VERSION="${VERSION:-${DEFAULT_VERSION}}" + + KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}" + KZG_FILE="${NOMOS_KZG_FILE:-kzgrs_test_params}" + KZG_CONTAINER_PATH="${NOMOS_KZG_CONTAINER_PATH:-/kzgrs_test_params/kzgrs_test_params}" + HOST_KZG_DIR="${ROOT_DIR}/${KZG_DIR_REL}" + HOST_KZG_FILE="${HOST_KZG_DIR}/${KZG_FILE}" +} + +run_examples::select_bin() { + case "${MODE}" in + compose) BIN="compose_runner" ;; + host) BIN="local_runner" ;; + k8s) BIN="k8s_runner" ;; + *) common::die "Unknown mode '${MODE}' (use compose|host|k8s)" ;; + esac +} + +run_examples::parse_args() { + MODE="compose" + RUN_SECS_RAW="" + DEMO_VALIDATORS="" + DEMO_EXECUTORS="" + IMAGE_SELECTION_MODE="auto" + + RUN_SECS_RAW_SPECIFIED="" + + while [ "$#" -gt 0 ]; do + case "$1" in + -h|--help) + run_examples::usage + exit 0 + ;; + -t|--run-seconds) + RUN_SECS_RAW_SPECIFIED=1 + RUN_SECS_RAW="${2:-}" + shift 2 + ;; + --run-seconds=*) + RUN_SECS_RAW_SPECIFIED=1 + RUN_SECS_RAW="${1#*=}" + shift + ;; + -v|--validators) + DEMO_VALIDATORS="${2:-}" + shift 2 + ;; + --validators=*) + DEMO_VALIDATORS="${1#*=}" + shift + ;; + -e|--executors) + DEMO_EXECUTORS="${2:-}" + shift 2 + ;; + --executors=*) + DEMO_EXECUTORS="${1#*=}" + shift + ;; + --bundle) + NOMOS_BINARIES_TAR="${2:-}" + export NOMOS_BINARIES_TAR + shift 2 + ;; + --bundle=*) + NOMOS_BINARIES_TAR="${1#*=}" + export NOMOS_BINARIES_TAR + shift + ;; + --local) + if [ "${IMAGE_SELECTION_MODE}" = "ecr" ]; then + run_examples::fail_with_usage "--local and --ecr are mutually exclusive" + fi + IMAGE_SELECTION_MODE="local" + shift + ;; + --ecr) + if [ "${IMAGE_SELECTION_MODE}" = "local" ]; then + run_examples::fail_with_usage "--local and --ecr are mutually exclusive" + fi + IMAGE_SELECTION_MODE="ecr" + shift + ;; + --no-image-build) + NOMOS_SKIP_IMAGE_BUILD=1 + export NOMOS_SKIP_IMAGE_BUILD + shift + ;; + compose|host|k8s) + MODE="$1" + shift + ;; + *) + # Positional run-seconds fallback for legacy usage. + if [ -z "${RUN_SECS_RAW_SPECIFIED}" ] && common::is_uint "$1"; then + RUN_SECS_RAW="$1" + shift + else + run_examples::fail_with_usage "Unknown argument: $1" + fi + ;; + esac + done + + if [ -n "${NOMOS_BINARIES_TAR:-}" ] && [ ! -f "${NOMOS_BINARIES_TAR}" ]; then + run_examples::fail_with_usage "NOMOS_BINARIES_TAR is set but missing: ${NOMOS_BINARIES_TAR}" + fi + + if ! common::is_uint "${RUN_SECS_RAW}" || [ "${RUN_SECS_RAW}" -le 0 ]; then + run_examples::fail_with_usage "run-seconds must be a positive integer (pass -t/--run-seconds)" + fi + RUN_SECS="${RUN_SECS_RAW}" + + if [ -z "${DEMO_VALIDATORS}" ] || [ -z "${DEMO_EXECUTORS}" ]; then + run_examples::fail_with_usage "validators and executors must be provided via -v/--validators and -e/--executors" + fi + if ! common::is_uint "${DEMO_VALIDATORS}" ; then + run_examples::fail_with_usage "validators must be a non-negative integer (pass -v/--validators)" + fi + if ! common::is_uint "${DEMO_EXECUTORS}" ; then + run_examples::fail_with_usage "executors must be a non-negative integer (pass -e/--executors)" + fi +} + +run_examples::select_image() { + local selection="${IMAGE_SELECTION_MODE}" + local context="" + + if [ "${selection}" = "auto" ]; then + if [ "${MODE}" = "k8s" ] && command -v kubectl >/dev/null 2>&1; then + context="$(kubectl config current-context 2>/dev/null || true)" + if [ "${context}" = "docker-desktop" ]; then + selection="local" + else + selection="ecr" + fi + else + selection="local" + fi + fi + + if [ "${selection}" = "local" ]; then + IMAGE="${NOMOS_TESTNET_IMAGE:-logos-blockchain-testing:local}" + export NOMOS_TESTNET_IMAGE_PULL_POLICY="${NOMOS_TESTNET_IMAGE_PULL_POLICY:-IfNotPresent}" + elif [ "${selection}" = "ecr" ]; then + local tag="${TAG:-test}" + if [ -n "${ECR_IMAGE:-}" ]; then + IMAGE="${ECR_IMAGE}" + elif [ -n "${ECR_REGISTRY:-}" ]; then + local registry="${ECR_REGISTRY}" + local repo="${ECR_REPO:-logos/logos-blockchain}" + IMAGE="${registry}/${repo}:${tag}" + elif [ -n "${AWS_ACCOUNT_ID:-}" ]; then + local aws_region="${AWS_REGION:-ap-southeast-2}" + local aws_account_id="${AWS_ACCOUNT_ID}" + local repo="${ECR_REPO:-logos-blockchain-testing}" + IMAGE="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/${repo}:${tag}" + else + local registry="public.ecr.aws/r4s5t9y4" + local repo="${ECR_REPO:-logos/logos-blockchain}" + IMAGE="${registry}/${repo}:${tag}" + fi + export NOMOS_TESTNET_IMAGE_PULL_POLICY="${NOMOS_TESTNET_IMAGE_PULL_POLICY:-Always}" + else + run_examples::fail_with_usage "Unknown image selection mode: ${selection}" + fi + + export IMAGE_TAG="${IMAGE}" + export NOMOS_TESTNET_IMAGE="${IMAGE}" + + if [ "${MODE}" = "k8s" ]; then + if [ "${selection}" = "ecr" ]; then + export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-inImage}" + else + export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-hostPath}" + fi + fi +} + +run_examples::default_tar_path() { + if [ -n "${NOMOS_BINARIES_TAR:-}" ]; then + echo "${NOMOS_BINARIES_TAR}" + return + fi + case "${MODE}" in + host) echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" ;; + compose|k8s) + if [ "${NOMOS_SKIP_IMAGE_BUILD:-}" = "1" ]; then + echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" + else + echo "${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz" + fi + ;; + *) echo "${ROOT_DIR}/.tmp/nomos-binaries-${VERSION}.tar.gz" ;; + esac +} + +run_examples::bundle_matches_expected() { + local tar_path="$1" + [ -f "${tar_path}" ] || return 1 + [ -z "${NOMOS_NODE_REV:-}" ] && return 0 + + local meta tar_rev tar_head + meta="$(tar -xOzf "${tar_path}" artifacts/nomos-bundle-meta.env 2>/dev/null || true)" + if [ -z "${meta}" ]; then + echo "Bundle meta missing in ${tar_path}; treating as stale and rebuilding." >&2 + return 1 + fi + tar_rev="$(echo "${meta}" | sed -n 's/^nomos_node_rev=//p' | head -n 1)" + tar_head="$(echo "${meta}" | sed -n 's/^nomos_node_git_head=//p' | head -n 1)" + if [ -n "${tar_rev}" ] && [ "${tar_rev}" != "${NOMOS_NODE_REV}" ]; then + echo "Bundle ${tar_path} is for nomos-node rev ${tar_rev}, expected ${NOMOS_NODE_REV}; rebuilding." >&2 + return 1 + fi + if [ -n "${tar_head}" ] && [ "${tar_head}" != "${NOMOS_NODE_REV}" ]; then + echo "Bundle ${tar_path} is for nomos-node git head ${tar_head}, expected ${NOMOS_NODE_REV}; rebuilding." >&2 + return 1 + fi + return 0 +} + +run_examples::host_bin_matches_arch() { + local bin_path="$1" + [ -x "${bin_path}" ] || return 1 + command -v file >/dev/null 2>&1 || return 0 + + local info expected + info="$(file -b "${bin_path}" 2>/dev/null || true)" + case "$(uname -m)" in + x86_64) expected="x86-64|x86_64" ;; + aarch64|arm64) expected="arm64|aarch64" ;; + *) expected="" ;; + esac + [ -n "${expected}" ] && echo "${info}" | grep -Eqi "${expected}" +} + +run_examples::restore_binaries_from_tar() { + local tar_path="${1:-}" + if [ -z "${tar_path}" ]; then + tar_path="$(run_examples::default_tar_path)" + fi + run_examples::bundle_matches_expected "${tar_path}" || return 1 + [ -f "${tar_path}" ] || return 1 + + local extract_dir="${ROOT_DIR}/.tmp/nomos-binaries" + echo "==> Restoring binaries from ${tar_path}" + rm -rf "${extract_dir}" + mkdir -p "${extract_dir}" + tar -xzf "${tar_path}" -C "${extract_dir}" || common::die "Failed to extract ${tar_path}" + + local src="${extract_dir}/artifacts" + local bin_dst="${ROOT_DIR}/testing-framework/assets/stack/bin" + local circuits_src="${src}/circuits" + local circuits_dst="${HOST_KZG_DIR}" + + RESTORED_BIN_DIR="${src}" + export RESTORED_BIN_DIR + + if [ ! -f "${src}/nomos-node" ] || [ ! -f "${src}/nomos-executor" ] || [ ! -f "${src}/nomos-cli" ]; then + echo "Binaries missing in ${tar_path}; provide a prebuilt binaries tarball." >&2 + return 1 + fi + + local copy_bins=1 + if [ "${MODE}" != "host" ] && ! run_examples::host_bin_matches_arch "${src}/nomos-node"; then + echo "Bundled binaries do not match host arch; skipping copy so containers rebuild from source." + copy_bins=0 + rm -f "${bin_dst}/nomos-node" "${bin_dst}/nomos-executor" "${bin_dst}/nomos-cli" + fi + if [ "${copy_bins}" -eq 1 ]; then + mkdir -p "${bin_dst}" + cp "${src}/nomos-node" "${src}/nomos-executor" "${src}/nomos-cli" "${bin_dst}/" + fi + + if [ -d "${circuits_src}" ] && [ -f "${circuits_src}/${KZG_FILE}" ]; then + rm -rf "${circuits_dst}" + mkdir -p "${circuits_dst}" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete "${circuits_src}/" "${circuits_dst}/" + else + rm -rf "${circuits_dst:?}/"* + cp -a "${circuits_src}/." "${circuits_dst}/" + fi + else + echo "Circuits missing in ${tar_path}; provide a prebuilt binaries/circuits tarball." >&2 + return 1 + fi + + RESTORED_BINARIES=1 + export RESTORED_BINARIES +} + +run_examples::ensure_binaries_tar() { + local platform="$1" + local tar_path="$2" + echo "==> Building fresh binaries bundle (${platform}) at ${tar_path}" + "${ROOT_DIR}/scripts/build-bundle.sh" --platform "${platform}" --output "${tar_path}" --rev "${NOMOS_NODE_REV}" +} + +run_examples::prepare_bundles() { + RESTORED_BINARIES=0 + NEED_HOST_RESTORE_AFTER_IMAGE=0 + + HOST_TAR="${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" + LINUX_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz" + + if [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN}" ]; then + echo "==> Using pre-specified host binaries (NOMOS_NODE_BIN/NOMOS_EXECUTOR_BIN); skipping tarball restore" + return 0 + fi + + # On non-Linux compose/k8s runs, use the Linux bundle for image build, then restore host bundle for the runner. + if [ "${MODE}" != "host" ] && [ "$(uname -s)" != "Linux" ] && [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "0" ] && [ -f "${LINUX_TAR}" ]; then + NEED_HOST_RESTORE_AFTER_IMAGE=1 + run_examples::restore_binaries_from_tar "${LINUX_TAR}" || { + run_examples::ensure_binaries_tar linux "${LINUX_TAR}" + run_examples::restore_binaries_from_tar "${LINUX_TAR}" + } + fi + + if ! run_examples::restore_binaries_from_tar; then + local tar_path + tar_path="$(run_examples::default_tar_path)" + case "${MODE}" in + host) run_examples::ensure_binaries_tar host "${tar_path}" ;; + compose|k8s) + if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then + run_examples::ensure_binaries_tar host "${tar_path}" + else + run_examples::ensure_binaries_tar linux "${tar_path}" + fi + ;; + *) run_examples::ensure_binaries_tar host "${tar_path}" ;; + esac + + run_examples::restore_binaries_from_tar "${tar_path}" || common::die \ + "Missing or invalid binaries tarball. Provide it via --bundle/NOMOS_BINARIES_TAR or place it at $(run_examples::default_tar_path)." + fi +} + +run_examples::maybe_rebuild_image() { + if [ "${MODE}" = "host" ]; then + return 0 + fi + + if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then + echo "==> Skipping testnet image rebuild (NOMOS_SKIP_IMAGE_BUILD=1)" + return 0 + fi + + echo "==> Rebuilding testnet image (${IMAGE})" + IMAGE_TAG="${IMAGE}" COMPOSE_CIRCUITS_PLATFORM="${COMPOSE_CIRCUITS_PLATFORM:-}" \ + "${ROOT_DIR}/testing-framework/assets/stack/scripts/build_test_image.sh" +} + +run_examples::maybe_restore_host_after_image() { + if [ "${NEED_HOST_RESTORE_AFTER_IMAGE}" != "1" ]; then + return 0 + fi + + echo "==> Restoring host bundle for runner (${HOST_TAR})" + if [ ! -f "${HOST_TAR}" ]; then + run_examples::ensure_binaries_tar host "${HOST_TAR}" + fi + run_examples::restore_binaries_from_tar "${HOST_TAR}" || common::die "Failed to restore host bundle from ${HOST_TAR}" +} + +run_examples::validate_restored_bundle() { + HOST_BUNDLE_PATH="${HOST_KZG_DIR}" + KZG_HOST_PATH="${HOST_BUNDLE_PATH}/${KZG_FILE}" + + if [ ! -x "${HOST_BUNDLE_PATH}/zksign/witness_generator" ]; then + common::die "Missing zksign/witness_generator in restored bundle; ensure the tarball contains host-compatible circuits." + fi + if [ ! -f "${KZG_HOST_PATH}" ]; then + common::die "KZG params missing at ${KZG_HOST_PATH}; ensure the tarball contains circuits." + fi + + if [ "${MODE}" = "host" ] && ! { [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN:-}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN:-}" ]; }; then + local tar_node tar_exec + tar_node="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-node" + tar_exec="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-executor" + + [ -x "${tar_node}" ] && [ -x "${tar_exec}" ] || common::die \ + "Restored tarball missing host executables; provide a host-compatible binaries tarball." + run_examples::host_bin_matches_arch "${tar_node}" && run_examples::host_bin_matches_arch "${tar_exec}" || common::die \ + "Restored executables do not match host architecture; provide a host-compatible binaries tarball." + + echo "==> Using restored host binaries from tarball" + NOMOS_NODE_BIN="${tar_node}" + NOMOS_EXECUTOR_BIN="${tar_exec}" + export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN + fi +} + +run_examples::kzg_path_for_mode() { + if [ "${MODE}" = "compose" ] || [ "${MODE}" = "k8s" ]; then + if [ "${MODE}" = "k8s" ] && [ "${NOMOS_KZG_MODE:-hostPath}" = "inImage" ]; then + echo "${NOMOS_KZG_IN_IMAGE_PARAMS_PATH:-/opt/nomos/kzg-params/kzgrs_test_params}" + else + echo "${KZG_CONTAINER_PATH}" + fi + else + echo "${KZG_HOST_PATH}" + fi +} + +run_examples::ensure_compose_circuits_platform_default() { + if [ "${MODE}" != "compose" ] || [ -n "${COMPOSE_CIRCUITS_PLATFORM:-}" ]; then + return 0 + fi + + local arch + arch="$(uname -m)" + case "${arch}" in + x86_64) COMPOSE_CIRCUITS_PLATFORM="linux-x86_64" ;; + arm64|aarch64) COMPOSE_CIRCUITS_PLATFORM="linux-aarch64" ;; + *) COMPOSE_CIRCUITS_PLATFORM="linux-x86_64" ;; + esac + export COMPOSE_CIRCUITS_PLATFORM +} + +run_examples::run() { + local kzg_path + kzg_path="$(run_examples::kzg_path_for_mode)" + + export NOMOS_DEMO_RUN_SECS="${RUN_SECS}" + export NOMOS_DEMO_VALIDATORS="${DEMO_VALIDATORS}" + export NOMOS_DEMO_EXECUTORS="${DEMO_EXECUTORS}" + + echo "==> Running ${BIN} for ${RUN_SECS}s (mode=${MODE}, image=${IMAGE})" + cd "${ROOT_DIR}" + + POL_PROOF_DEV_MODE=true \ + TESTNET_PRINT_ENDPOINTS=1 \ + NOMOS_TESTNET_IMAGE="${IMAGE}" \ + NOMOS_CIRCUITS="${HOST_BUNDLE_PATH}" \ + NOMOS_KZGRS_PARAMS_PATH="${kzg_path}" \ + NOMOS_NODE_BIN="${NOMOS_NODE_BIN:-}" \ + NOMOS_EXECUTOR_BIN="${NOMOS_EXECUTOR_BIN:-}" \ + COMPOSE_CIRCUITS_PLATFORM="${COMPOSE_CIRCUITS_PLATFORM:-}" \ + cargo run -p runner-examples --bin "${BIN}" +} + +run_examples::main() { + run_examples::load_env + run_examples::parse_args "$@" + run_examples::select_bin + run_examples::select_image + + run_examples::prepare_bundles + echo "==> Using restored circuits/binaries bundle" + + SETUP_OUT="$(common::tmpfile nomos-setup-output.XXXXXX)" + cleanup() { rm -f "${SETUP_OUT}" 2>/dev/null || true; } + trap cleanup EXIT + + run_examples::maybe_rebuild_image + run_examples::maybe_restore_host_after_image + run_examples::validate_restored_bundle + run_examples::ensure_compose_circuits_platform_default + run_examples::run +} diff --git a/scripts/lib/setup-circuits-stack.sh b/scripts/lib/setup-circuits-stack.sh new file mode 100644 index 0000000..a118cbd --- /dev/null +++ b/scripts/lib/setup-circuits-stack.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Intended to be sourced by scripts/setup-circuits-stack.sh +# shellcheck disable=SC1091 +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh" + +setup_circuits_stack::usage() { + cat <<'EOF' +Usage: scripts/setup-circuits-stack.sh [VERSION] + +Prepares circuits for both the Docker image (Linux/x86_64) and the host (for +witness generators). + +Env overrides: + STACK_DIR Where to place the Linux bundle (default: testing-framework/assets/stack/kzgrs_test_params) + HOST_DIR Where to place the host bundle (default: .tmp/nomos-circuits-host) + LINUX_STAGE_DIR Optional staging dir for the Linux bundle (default: .tmp/nomos-circuits-linux) + NOMOS_CIRCUITS_PLATFORM Force host platform (e.g., macos-aarch64) + NOMOS_CIRCUITS_REBUILD_RAPIDSNARK Set to 1 to force rebuild (host bundle only) +EOF +} + +setup_circuits_stack::fail_with_usage() { + echo "$1" >&2 + setup_circuits_stack::usage + exit 1 +} + +setup_circuits_stack::realpath_py() { + python3 - "$1" <<'PY' +import os, sys +print(os.path.realpath(sys.argv[1])) +PY +} + +setup_circuits_stack::detect_platform() { + local os arch + case "$(uname -s)" in + Linux*) os="linux" ;; + Darwin*) os="macos" ;; + MINGW*|MSYS*|CYGWIN*) os="windows" ;; + *) common::die "Unsupported OS: $(uname -s)" ;; + esac + + case "$(uname -m)" in + x86_64) arch="x86_64" ;; + aarch64|arm64) arch="aarch64" ;; + *) common::die "Unsupported arch: $(uname -m)" ;; + esac + + echo "${os}-${arch}" +} + +setup_circuits_stack::fetch_bundle() { + local platform="$1" + local dest="$2" + local rebuild="${3:-0}" + + rm -rf "${dest}" + mkdir -p "${dest}" + + NOMOS_CIRCUITS_PLATFORM="${platform}" \ + NOMOS_CIRCUITS_REBUILD_RAPIDSNARK="${rebuild}" \ + "${ROOT_DIR}/scripts/setup-nomos-circuits.sh" "${VERSION}" "${dest}" +} + +setup_circuits_stack::fetch_kzg_params() { + local dest_dir="$1" + local dest_file="${dest_dir}/${KZG_FILE}" + local url="https://raw.githubusercontent.com/logos-co/nomos-node/${NOMOS_NODE_REV}/tests/kzgrs/kzgrs_test_params" + + echo "Fetching KZG parameters from ${url}" + curl -fsSL "${url}" -o "${dest_file}" +} + +setup_circuits_stack::load_env() { + ROOT_DIR="$(common::repo_root)" + export ROOT_DIR + + common::require_file "${ROOT_DIR}/versions.env" + # shellcheck disable=SC1091 + . "${ROOT_DIR}/versions.env" + common::maybe_source "${ROOT_DIR}/paths.env" + + KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}" + KZG_FILE="${NOMOS_KZG_FILE:-kzgrs_test_params}" + HOST_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_HOST_DIR_REL:-.tmp/nomos-circuits-host}" + LINUX_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_LINUX_DIR_REL:-.tmp/nomos-circuits-linux}" + + VERSION="${VERSION:-v0.3.1}" + STACK_DIR="${STACK_DIR:-${ROOT_DIR}/${KZG_DIR_REL}}" + HOST_DIR="${HOST_DIR:-${ROOT_DIR}/${HOST_DIR_REL_DEFAULT}}" + LINUX_STAGE_DIR="${LINUX_STAGE_DIR:-${ROOT_DIR}/${LINUX_DIR_REL_DEFAULT}}" + + NOMOS_NODE_REV="${NOMOS_NODE_REV:?Missing NOMOS_NODE_REV in versions.env or env}" + + # Force non-interactive installs so repeated runs do not prompt. + export NOMOS_CIRCUITS_NONINTERACTIVE=1 +} + +setup_circuits_stack::main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + setup_circuits_stack::usage + exit 0 + fi + + setup_circuits_stack::load_env + if [ -n "${1:-}" ]; then + VERSION="$1" + fi + + echo "Preparing circuits (version ${VERSION})" + echo "Workspace: ${ROOT_DIR}" + + local linux_platform="linux-x86_64" + + echo "Installing Linux bundle for Docker image into ${STACK_DIR}" + local stage_real stack_real + stage_real="$(setup_circuits_stack::realpath_py "${LINUX_STAGE_DIR}")" + stack_real="$(setup_circuits_stack::realpath_py "${STACK_DIR}")" + + if [ "${stage_real}" = "${stack_real}" ]; then + rm -rf "${STACK_DIR}" + setup_circuits_stack::fetch_bundle "${linux_platform}" "${STACK_DIR}" 0 + setup_circuits_stack::fetch_kzg_params "${STACK_DIR}" + else + rm -rf "${LINUX_STAGE_DIR}" + mkdir -p "${LINUX_STAGE_DIR}" + setup_circuits_stack::fetch_bundle "${linux_platform}" "${LINUX_STAGE_DIR}" 0 + rm -rf "${STACK_DIR}" + mkdir -p "${STACK_DIR}" + cp -R "${LINUX_STAGE_DIR}/." "${STACK_DIR}/" + setup_circuits_stack::fetch_kzg_params "${STACK_DIR}" + fi + echo "Linux bundle ready at ${STACK_DIR}" + + local host_platform + host_platform="${NOMOS_CIRCUITS_PLATFORM:-$(setup_circuits_stack::detect_platform)}" + if [[ "${host_platform}" == "${linux_platform}" ]]; then + echo "Host platform ${host_platform} matches Linux bundle; host can reuse ${STACK_DIR}" + echo "Export if you want to be explicit:" + echo " export NOMOS_CIRCUITS=\"${STACK_DIR}\"" + else + echo "Host platform detected: ${host_platform}; installing host-native bundle into ${HOST_DIR}" + setup_circuits_stack::fetch_bundle "${host_platform}" "${HOST_DIR}" "${NOMOS_CIRCUITS_REBUILD_RAPIDSNARK:-0}" + setup_circuits_stack::fetch_kzg_params "${HOST_DIR}" + echo "Host bundle ready at ${HOST_DIR}" + echo + echo "Set for host runs:" + echo " export NOMOS_CIRCUITS=\"${HOST_DIR}\"" + fi + + cat <<'EOF' + +Done. +- For Docker/compose: rebuild the image to bake the Linux bundle: + testing-framework/assets/stack/scripts/build_test_image.sh +- For host runs (e.g., compose_runner): ensure NOMOS_CIRCUITS points to the host bundle above. +EOF +} + diff --git a/scripts/lib/setup-nomos-circuits.sh b/scripts/lib/setup-nomos-circuits.sh new file mode 100755 index 0000000..2ec0838 --- /dev/null +++ b/scripts/lib/setup-nomos-circuits.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +set -euo pipefail + +setup_nomos_circuits::usage() { + cat <<'EOF' +Usage: scripts/setup-nomos-circuits.sh [VERSION] [INSTALL_DIR] + +Arguments: + VERSION Optional. Version to install (default: v0.3.1) + INSTALL_DIR Optional. Installation directory (default: $HOME/.nomos-circuits) + +Environment: + NOMOS_CIRCUITS_PLATFORM Override platform (e.g. linux-x86_64, macos-aarch64) + NOMOS_CIRCUITS_NONINTERACTIVE Set to 1 to auto-overwrite without prompt + NOMOS_CIRCUITS_REBUILD_RAPIDSNARK Set to 1 to force rapidsnark rebuild + GITHUB_TOKEN Optional token for GitHub releases download +EOF +} + +setup_nomos_circuits::init_vars() { + VERSION="${1:-v0.3.1}" + DEFAULT_INSTALL_DIR="${HOME}/.nomos-circuits" + INSTALL_DIR="${2:-${DEFAULT_INSTALL_DIR}}" + REPO="logos-co/nomos-circuits" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + NONINTERACTIVE="${NOMOS_CIRCUITS_NONINTERACTIVE:-0}" + + # Colors for output + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + NC='\033[0m' # No Color +} + +setup_nomos_circuits::print_info() { echo -e "${BLUE}ℹ${NC} $1"; } +setup_nomos_circuits::print_success() { echo -e "${GREEN}✓${NC} $1"; } +setup_nomos_circuits::print_warning() { echo -e "${YELLOW}⚠${NC} $1"; } +setup_nomos_circuits::print_error() { echo -e "${RED}✗${NC} $1"; } + +setup_nomos_circuits::detect_platform() { + local os="" arch="" + + case "$(uname -s)" in + Linux*) os="linux" ;; + Darwin*) os="macos" ;; + MINGW*|MSYS*|CYGWIN*) os="windows" ;; + *) setup_nomos_circuits::print_error "Unsupported operating system: $(uname -s)"; exit 1 ;; + esac + + case "$(uname -m)" in + x86_64) arch="x86_64" ;; + aarch64|arm64) arch="aarch64" ;; + *) setup_nomos_circuits::print_error "Unsupported architecture: $(uname -m)"; exit 1 ;; + esac + + echo "${os}-${arch}" +} + +setup_nomos_circuits::check_existing_installation() { + if [ -d "${INSTALL_DIR}" ]; then + setup_nomos_circuits::print_warning "Installation directory already exists: ${INSTALL_DIR}" + + if [ -f "${INSTALL_DIR}/VERSION" ]; then + local current_version + current_version="$(cat "${INSTALL_DIR}/VERSION")" + setup_nomos_circuits::print_info "Currently installed version: ${current_version}" + fi + + if [ "${NONINTERACTIVE}" = "1" ] || [ ! -t 0 ]; then + setup_nomos_circuits::print_info "Non-interactive environment detected, automatically overwriting..." + else + echo + read -p "Do you want to overwrite it? (y/N): " -n 1 -r + echo + if [[ ! ${REPLY} =~ ^[Yy]$ ]]; then + setup_nomos_circuits::print_info "Installation cancelled." + exit 0 + fi + fi + + setup_nomos_circuits::print_info "Removing existing installation..." + rm -rf "${INSTALL_DIR}" + fi +} + +setup_nomos_circuits::download_release() { + local platform="$1" + local artifact="nomos-circuits-${VERSION}-${platform}.tar.gz" + local url="https://github.com/${REPO}/releases/download/${VERSION}/${artifact}" + local temp_dir + temp_dir="$(mktemp -d)" + + setup_nomos_circuits::print_info "Downloading nomos-circuits ${VERSION} for ${platform}..." + setup_nomos_circuits::print_info "URL: ${url}" + + local curl_cmd="curl -fL --retry 5 --retry-delay 2 --retry-all-errors" + if [ -n "${GITHUB_TOKEN:-}" ]; then + curl_cmd="${curl_cmd} --header 'authorization: Bearer ${GITHUB_TOKEN}'" + fi + curl_cmd="${curl_cmd} -o ${temp_dir}/${artifact} ${url}" + + if ! eval "${curl_cmd}"; then + setup_nomos_circuits::print_error "Failed to download release artifact" + setup_nomos_circuits::print_error "Please check that version ${VERSION} exists for platform ${platform}" + setup_nomos_circuits::print_error "Available releases: https://github.com/${REPO}/releases" + rm -rf "${temp_dir}" + return 1 + fi + + setup_nomos_circuits::print_success "Download complete" + + if ! tar -tzf "${temp_dir}/${artifact}" >/dev/null 2>&1; then + setup_nomos_circuits::print_error "Downloaded archive is not a valid tar.gz: ${temp_dir}/${artifact}" + rm -rf "${temp_dir}" + return 1 + fi + + setup_nomos_circuits::print_info "Extracting to ${INSTALL_DIR}..." + mkdir -p "${INSTALL_DIR}" + + if ! tar -xzf "${temp_dir}/${artifact}" -C "${INSTALL_DIR}" --strip-components=1; then + setup_nomos_circuits::print_error "Failed to extract archive" + rm -rf "${temp_dir}" + return 1 + fi + + rm -rf "${temp_dir}" + setup_nomos_circuits::print_success "Extraction complete" +} + +setup_nomos_circuits::handle_macos_quarantine() { + setup_nomos_circuits::print_info "macOS detected: Removing quarantine attributes from executables..." + + if find "${INSTALL_DIR}" -type f -perm -111 -exec xattr -d com.apple.quarantine {} \; 2>/dev/null; then + setup_nomos_circuits::print_success "Quarantine attributes removed" + else + setup_nomos_circuits::print_warning "Could not remove quarantine attributes (they may not exist)" + fi +} + +setup_nomos_circuits::print_circuits() { + setup_nomos_circuits::print_info "The following circuits are available:" + local dir circuit_name + for dir in "${INSTALL_DIR}"/*/; do + if [ -d "${dir}" ]; then + circuit_name="$(basename "${dir}")" + if [ -f "${dir}/witness_generator" ]; then + echo " • ${circuit_name}" + fi + fi + done +} + +setup_nomos_circuits::resolve_platform() { + local platform_override="${NOMOS_CIRCUITS_PLATFORM:-}" + if [ -n "${platform_override}" ]; then + PLATFORM="${platform_override}" + setup_nomos_circuits::print_info "Using overridden platform: ${PLATFORM}" + else + PLATFORM="$(setup_nomos_circuits::detect_platform)" + setup_nomos_circuits::print_info "Detected platform: ${PLATFORM}" + fi +} + +setup_nomos_circuits::download_with_fallbacks() { + # Outputs: + # PLATFORM - platform used for the downloaded bundle + # REBUILD_REQUIRED - 0/1 + REBUILD_REQUIRED="${NOMOS_CIRCUITS_REBUILD_RAPIDSNARK:-0}" + + if setup_nomos_circuits::download_release "${PLATFORM}"; then + return 0 + fi + + if [[ "${PLATFORM}" == "linux-aarch64" ]]; then + setup_nomos_circuits::print_warning "Falling back to linux-x86_64 circuits bundle; will rebuild prover for aarch64." + rm -rf "${INSTALL_DIR}" + PLATFORM="linux-x86_64" + setup_nomos_circuits::download_release "${PLATFORM}" || return 1 + REBUILD_REQUIRED=1 + return 0 + fi + + if [[ "${PLATFORM}" == "macos-x86_64" ]]; then + setup_nomos_circuits::print_warning "No macOS x86_64 bundle; falling back to macOS aarch64 circuits bundle and rebuilding prover." + rm -rf "${INSTALL_DIR}" + PLATFORM="macos-aarch64" + if ! setup_nomos_circuits::download_release "${PLATFORM}"; then + setup_nomos_circuits::print_warning "macOS aarch64 bundle unavailable; trying linux-x86_64 bundle and rebuilding prover." + rm -rf "${INSTALL_DIR}" + PLATFORM="linux-x86_64" + setup_nomos_circuits::download_release "${PLATFORM}" || return 1 + fi + REBUILD_REQUIRED=1 + return 0 + fi + + return 1 +} + +setup_nomos_circuits::maybe_handle_quarantine() { + if [[ "${PLATFORM}" == macos-* ]]; then + echo + setup_nomos_circuits::handle_macos_quarantine + fi +} + +setup_nomos_circuits::maybe_rebuild_rapidsnark() { + if [[ "${REBUILD_REQUIRED}" == "1" ]]; then + echo + setup_nomos_circuits::print_info "Rebuilding rapidsnark prover for ${PLATFORM}..." + "${SCRIPT_DIR}/build-rapidsnark.sh" "${INSTALL_DIR}" + else + setup_nomos_circuits::print_info "Skipping rapidsnark rebuild (set NOMOS_CIRCUITS_REBUILD_RAPIDSNARK=1 to force)." + fi +} + +setup_nomos_circuits::print_summary() { + echo + setup_nomos_circuits::print_success "Installation complete!" + echo + setup_nomos_circuits::print_info "nomos-circuits ${VERSION} is now installed at: ${INSTALL_DIR}" + setup_nomos_circuits::print_circuits + + if [ "${INSTALL_DIR}" != "${DEFAULT_INSTALL_DIR}" ]; then + echo + setup_nomos_circuits::print_info "Since you're using a custom installation directory, set the environment variable:" + setup_nomos_circuits::print_info " export NOMOS_CIRCUITS=${INSTALL_DIR}" + echo + fi +} + +setup_nomos_circuits::main() { + if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + setup_nomos_circuits::usage + exit 0 + fi + + setup_nomos_circuits::init_vars "${1:-}" "${2:-}" + + setup_nomos_circuits::print_info "Setting up nomos-circuits ${VERSION}" + setup_nomos_circuits::print_info "Installation directory: ${INSTALL_DIR}" + echo + + setup_nomos_circuits::resolve_platform + + setup_nomos_circuits::check_existing_installation + + setup_nomos_circuits::download_with_fallbacks || exit 1 + setup_nomos_circuits::maybe_handle_quarantine + setup_nomos_circuits::maybe_rebuild_rapidsnark + setup_nomos_circuits::print_summary +} diff --git a/scripts/lib/update-nomos-rev.sh b/scripts/lib/update-nomos-rev.sh new file mode 100644 index 0000000..fb38f7f --- /dev/null +++ b/scripts/lib/update-nomos-rev.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Intended to be sourced by scripts/update-nomos-rev.sh +# shellcheck disable=SC1091 +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh" + +update_nomos_rev::usage() { + cat <<'EOF' +Usage: + scripts/update-nomos-rev.sh --rev + scripts/update-nomos-rev.sh --path + scripts/update-nomos-rev.sh --unskip-worktree + +Notes: + --rev sets NOMOS_NODE_REV and updates Cargo.toml revs + --path sets NOMOS_NODE_PATH (clears NOMOS_NODE_REV) and patches Cargo.toml to use a local nomos-node checkout + --unskip-worktree clears any skip-worktree flag for Cargo.toml + Only one may be used at a time. +EOF +} + +update_nomos_rev::fail_with_usage() { + echo "$1" >&2 + update_nomos_rev::usage + exit 1 +} + +update_nomos_rev::maybe_unskip_worktree() { + local file="$1" + if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git -C "${ROOT_DIR}" update-index --no-skip-worktree "${file}" >/dev/null 2>&1 || true + fi +} + +update_nomos_rev::maybe_skip_worktree() { + local file="$1" + if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git -C "${ROOT_DIR}" update-index --skip-worktree "${file}" >/dev/null 2>&1 || true + fi +} + +update_nomos_rev::ensure_env_key() { + local key="$1" default_value="$2" + if ! grep -Eq "^#?[[:space:]]*${key}=" "${ROOT_DIR}/versions.env"; then + echo "${default_value}" >> "${ROOT_DIR}/versions.env" + fi +} + +update_nomos_rev::parse_args() { + REV="" + LOCAL_PATH="" + UNSKIP_WORKTREE=0 + + while [ "$#" -gt 0 ]; do + case "$1" in + --rev) REV="${2:-}"; shift 2 ;; + --path) LOCAL_PATH="${2:-}"; shift 2 ;; + --unskip-worktree) UNSKIP_WORKTREE=1; shift ;; + -h|--help) update_nomos_rev::usage; exit 0 ;; + *) update_nomos_rev::fail_with_usage "Unknown arg: $1" ;; + esac + done + + if [ "${UNSKIP_WORKTREE}" -eq 1 ] && { [ -n "${REV}" ] || [ -n "${LOCAL_PATH}" ]; }; then + update_nomos_rev::fail_with_usage "Use --unskip-worktree alone." + fi + if [ -n "${REV}" ] && [ -n "${LOCAL_PATH}" ]; then + update_nomos_rev::fail_with_usage "Use either --rev or --path, not both" + fi + if [ -z "${REV}" ] && [ -z "${LOCAL_PATH}" ] && [ "${UNSKIP_WORKTREE}" -eq 0 ]; then + update_nomos_rev::usage + exit 1 + fi +} + +update_nomos_rev::load_env() { + ROOT_DIR="$(common::repo_root)" + export ROOT_DIR + common::require_file "${ROOT_DIR}/versions.env" +} + +update_nomos_rev::update_to_rev() { + local rev="$1" + echo "Updating nomos-node rev to ${rev}" + + sed -i.bak -E \ + -e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/NOMOS_NODE_REV=${rev}/" \ + -e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/# NOMOS_NODE_PATH=/" \ + "${ROOT_DIR}/versions.env" + rm -f "${ROOT_DIR}/versions.env.bak" + + python3 - "${ROOT_DIR}" "${rev}" <<'PY' +import pathlib, re, sys +root = pathlib.Path(sys.argv[1]) +rev = sys.argv[2] +cargo_toml = root / "Cargo.toml" +txt = cargo_toml.read_text() +txt = txt.replace("\\n", "\n") +txt = re.sub( + r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)', + "", + txt, +) +txt = re.sub( + r'(git = "https://github\.com/logos-co/nomos-node\.git", rev = ")[^"]+(")', + r"\g<1>" + rev + r"\2", + txt, +) +cargo_toml.write_text(txt.rstrip() + "\n") +PY + + update_nomos_rev::maybe_unskip_worktree "Cargo.toml" +} + +update_nomos_rev::update_to_path() { + local node_path="$1" + echo "Pointing to local nomos-node at ${node_path}" + + [ -d "${node_path}" ] || common::die "path does not exist: ${node_path}" + + local current_rev escaped_path + current_rev="$(grep -E '^[#[:space:]]*NOMOS_NODE_REV=' "${ROOT_DIR}/versions.env" | head -n1 | sed -E 's/^#?[[:space:]]*NOMOS_NODE_REV=//')" + escaped_path="${node_path//\//\\/}" + + sed -i.bak -E \ + -e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/NOMOS_NODE_PATH=${escaped_path}/" \ + -e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/# NOMOS_NODE_REV=${current_rev}/" \ + "${ROOT_DIR}/versions.env" + rm -f "${ROOT_DIR}/versions.env.bak" + + local python_bin="${PYTHON_BIN:-python3}" + command -v "${python_bin}" >/dev/null 2>&1 || common::die "python3 is required to patch Cargo.toml for local paths" + + "${python_bin}" - "${ROOT_DIR}" "${node_path}" <<'PY' +import json +import pathlib +import re +import subprocess +import sys + +root = pathlib.Path(sys.argv[1]) +node_path = pathlib.Path(sys.argv[2]) + +targets = [ + "broadcast-service", "chain-leader", "chain-network", "chain-service", + "common-http-client", "cryptarchia-engine", "cryptarchia-sync", + "executor-http-client", "groth16", "key-management-system-service", + "kzgrs", "kzgrs-backend", "nomos-api", "nomos-blend-message", + "nomos-blend-service", "nomos-core", "nomos-da-dispersal", + "nomos-da-network-core", "nomos-da-network-service", "nomos-da-sampling", + "nomos-da-verifier", "nomos-executor", "nomos-http-api-common", + "nomos-ledger", "nomos-libp2p", "nomos-network", "nomos-node", + "nomos-sdp", "nomos-time", "nomos-tracing", "nomos-tracing-service", + "nomos-utils", "nomos-wallet", "poc", "pol", "subnetworks-assignations", + "tests", "tx-service", "wallet", "zksign", +] + +try: + meta = subprocess.check_output( + ["cargo", "metadata", "--format-version", "1", "--no-deps"], + cwd=node_path, + ) +except subprocess.CalledProcessError as exc: + sys.stderr.write(f"Failed to run cargo metadata in {node_path}: {exc}\n") + sys.exit(1) + +data = json.loads(meta) +paths = {} +for pkg in data.get("packages", []): + paths[pkg["name"]] = str(pathlib.Path(pkg["manifest_path"]).parent) + +patch_lines = ['[patch."https://github.com/logos-co/nomos-node"]'] +missing = [] +for name in targets: + if name in paths: + patch_lines.append(f'{name} = {{ path = "{paths[name]}" }}') + else: + missing.append(name) + +cargo_toml = root / "Cargo.toml" +txt = cargo_toml.read_text() +txt = txt.replace("\\n", "\n") +txt = re.sub( + r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)', + "", + txt, +) +txt = txt.rstrip() + "\n\n" + "\n".join(patch_lines) + "\n" +cargo_toml.write_text(txt) + +if missing: + sys.stderr.write( + "Warning: missing crates in local nomos-node checkout: " + + ", ".join(missing) + + "\n" + ) +PY + + update_nomos_rev::maybe_skip_worktree "Cargo.toml" + echo "Local nomos-node patch applied; Cargo.toml marked skip-worktree (run --unskip-worktree to clear)." +} + +update_nomos_rev::main() { + update_nomos_rev::load_env + update_nomos_rev::parse_args "$@" + + update_nomos_rev::ensure_env_key "NOMOS_NODE_REV" "# NOMOS_NODE_REV=" + update_nomos_rev::ensure_env_key "NOMOS_NODE_PATH" "# NOMOS_NODE_PATH=" + + if [ "${UNSKIP_WORKTREE}" -eq 1 ]; then + update_nomos_rev::maybe_unskip_worktree "Cargo.toml" + echo "Cleared skip-worktree on Cargo.toml (if it was set)." + exit 0 + fi + + if [ -n "${REV}" ]; then + update_nomos_rev::update_to_rev "${REV}" + else + update_nomos_rev::update_to_path "${LOCAL_PATH}" + fi + + echo "Done. Consider updating Cargo.lock if needed (cargo fetch)." +} + diff --git a/scripts/run-examples.sh b/scripts/run-examples.sh index 75f6e5f..975d5d2 100755 --- a/scripts/run-examples.sh +++ b/scripts/run-examples.sh @@ -1,512 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -# All-in-one helper: prepare circuits (Linux + host), rebuild the image, and run -# the chosen runner binary. -# -# Usage: scripts/run-examples.sh [options] [compose|host|k8s] -# compose -> runs examples/src/bin/compose_runner.rs (default) -# host -> runs examples/src/bin/local_runner.rs -# k8s -> runs examples/src/bin/k8s_runner.rs -# run-seconds must be provided via -t/--run-seconds -# -# Env overrides: -# VERSION - circuits version (default v0.3.1) -# NOMOS_TESTNET_IMAGE - image reference (overridden by --ecr/--local selection) -# ECR_IMAGE - full image reference to use for --ecr (overrides ECR_REGISTRY/ECR_REPO/TAG) -# ECR_REGISTRY - registry hostname for --ecr (default public.ecr.aws/r4s5t9y4) -# ECR_REPO - repository path for --ecr (default logos/logos-blockchain) -# TAG - tag for --ecr (default test) -# AWS_REGION - (legacy) private ECR region; used only if AWS_ACCOUNT_ID is set and ECR_REGISTRY is unset -# AWS_ACCOUNT_ID - (legacy) private ECR account id; used only if ECR_REGISTRY is unset -# NOMOS_TESTNET_IMAGE_PULL_POLICY - k8s imagePullPolicy (default IfNotPresent; set to Always for --ecr) -# NOMOS_CIRCUITS_PLATFORM - override host platform detection -# NOMOS_CIRCUITS_REBUILD_RAPIDSNARK - set to 1 to force rapidsnark rebuild -# NOMOS_BINARIES_TAR - path to prebuilt binaries/circuits tarball (required; default .tmp/nomos-binaries--.tar.gz) - -usage() { - cat <<'EOF' -Usage: scripts/run-examples.sh [options] [compose|host|k8s] - -Modes: - compose Run examples/src/bin/compose_runner.rs (default) - host Run examples/src/bin/local_runner.rs - k8s Run examples/src/bin/k8s_runner.rs - -Options: - -t, --run-seconds N Duration to run the demo (required) - -v, --validators N Number of validators (required) - -e, --executors N Number of executors (required) - --local Use a local Docker image tag (default for docker-desktop k8s) - --ecr Use an ECR image reference (default for non-docker-desktop k8s) - --no-image-build Skip rebuilding the compose/k8s image (sets NOMOS_SKIP_IMAGE_BUILD=1) - -Environment: - VERSION Circuits version (default v0.3.1) - NOMOS_TESTNET_IMAGE Image reference (overridden by --local/--ecr selection) - ECR_IMAGE Full image reference for --ecr (overrides ECR_REGISTRY/ECR_REPO/TAG) - ECR_REGISTRY Registry hostname for --ecr (default public.ecr.aws/r4s5t9y4) - ECR_REPO Repository path for --ecr (default logos/logos-blockchain) - TAG Tag for --ecr (default test) - AWS_REGION (legacy) Private ECR region (used only if AWS_ACCOUNT_ID is set and ECR_REGISTRY is unset) - AWS_ACCOUNT_ID (legacy) Private ECR account id (used only if ECR_REGISTRY is unset) - NOMOS_TESTNET_IMAGE_PULL_POLICY K8s imagePullPolicy (default IfNotPresent; set to Always for --ecr) - NOMOS_CIRCUITS_PLATFORM Override host platform detection - NOMOS_CIRCUITS_REBUILD_RAPIDSNARK Force rapidsnark rebuild - NOMOS_BINARIES_TAR Path to prebuilt binaries/circuits tarball (required) - NOMOS_SKIP_IMAGE_BUILD Set to 1 to skip rebuilding the compose/k8s image - TESTNET_PRINT_ENDPOINTS If set, runners print TESTNET_ENDPOINTS/TESTNET_PPROF (set automatically) - COMPOSE_RUNNER_HTTP_TIMEOUT_SECS Compose readiness timeout override - K8S_RUNNER_DEPLOYMENT_TIMEOUT_SECS K8s deployment readiness timeout override - K8S_RUNNER_HTTP_TIMEOUT_SECS K8s port-forward readiness timeout override - K8S_RUNNER_HTTP_PROBE_TIMEOUT_SECS K8s NodePort readiness timeout override - K8S_RUNNER_PROMETHEUS_HTTP_TIMEOUT_SECS K8s Prometheus port-forward readiness timeout override - K8S_RUNNER_PROMETHEUS_HTTP_PROBE_TIMEOUT_SECS K8s Prometheus NodePort probe timeout override -EOF -} - -fail_with_usage() { - echo "$1" >&2 - usage - exit 1 -} - -if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then - usage - exit 0 +# Thin wrapper; the actual implementation lives in scripts/lib/run-examples.sh +# so it can be shared/tested more easily. +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" fi -# If a tarball is explicitly provided, ensure it exists before doing work. -if [ -n "${NOMOS_BINARIES_TAR:-}" ] && [ ! -f "${NOMOS_BINARIES_TAR}" ]; then - fail_with_usage "NOMOS_BINARIES_TAR is set but missing: ${NOMOS_BINARIES_TAR}" -fi +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -readonly ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [ ! -f "${ROOT_DIR}/versions.env" ]; then - echo "ERROR: versions.env missing; run from repo root or restore the file." >&2 - exit 1 -fi # shellcheck disable=SC1091 -. "${ROOT_DIR}/versions.env" -if [ -f "${ROOT_DIR}/paths.env" ]; then - # shellcheck disable=SC1091 - . "${ROOT_DIR}/paths.env" -fi -readonly DEFAULT_VERSION="${VERSION:?Missing VERSION in versions.env}" -readonly KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}" -readonly KZG_FILE="${NOMOS_KZG_FILE:-kzgrs_test_params}" -readonly KZG_CONTAINER_PATH="${NOMOS_KZG_CONTAINER_PATH:-/kzgrs_test_params/kzgrs_test_params}" -readonly HOST_KZG_DIR="${ROOT_DIR}/${KZG_DIR_REL}" -readonly HOST_KZG_FILE="${HOST_KZG_DIR}/${KZG_FILE}" -MODE="compose" -RUN_SECS_RAW="" -VERSION="${DEFAULT_VERSION}" -IMAGE_SELECTION_MODE="auto" -IMAGE="" -DEMO_VALIDATORS="" -DEMO_EXECUTORS="" -while [ "$#" -gt 0 ]; do - case "$1" in - -h|--help) - usage; exit 0 ;; - -t|--run-seconds) - RUN_SECS_RAW="${2:-}"; shift 2 ;; - -v|--validators) - DEMO_VALIDATORS="${2:-}"; shift 2 ;; - -e|--executors) - DEMO_EXECUTORS="${2:-}"; shift 2 ;; - --local) - if [ "${IMAGE_SELECTION_MODE}" = "ecr" ]; then - fail_with_usage "--local and --ecr are mutually exclusive" - fi - IMAGE_SELECTION_MODE="local" - shift ;; - --ecr) - if [ "${IMAGE_SELECTION_MODE}" = "local" ]; then - fail_with_usage "--local and --ecr are mutually exclusive" - fi - IMAGE_SELECTION_MODE="ecr" - shift ;; - --no-image-build) - NOMOS_SKIP_IMAGE_BUILD=1 - export NOMOS_SKIP_IMAGE_BUILD - shift ;; - compose|host|k8s) - MODE="$1"; shift ;; - *) - # Positional run-seconds fallback for legacy usage - if [ -z "${RUN_SECS_RAW_SPECIFIED:-}" ] && [[ "$1" =~ ^[0-9]+$ ]]; then - RUN_SECS_RAW="$1" - shift - else - fail_with_usage "Unknown argument: $1" - fi - ;; - esac -done -RESTORED_BINARIES=0 -SETUP_OUT="" -cleanup() { - if [ -n "${SETUP_OUT}" ]; then - rm -f "${SETUP_OUT}" - fi -} -trap cleanup EXIT +. "${ROOT_DIR}/scripts/lib/run-examples.sh" -case "$MODE" in - compose) BIN="compose_runner" ;; - host) BIN="local_runner" ;; - k8s) BIN="k8s_runner" ;; - *) echo "Unknown mode '$MODE' (use compose|host|k8s)" >&2; exit 1 ;; -esac - -select_image() { - local selection="${IMAGE_SELECTION_MODE}" - local context="" - - if [ "${selection}" = "auto" ]; then - if [ "${MODE}" = "k8s" ] && command -v kubectl >/dev/null 2>&1; then - context="$(kubectl config current-context 2>/dev/null || true)" - if [ "${context}" = "docker-desktop" ]; then - selection="local" - else - selection="ecr" - fi - else - selection="local" - fi - fi - - if [ "${selection}" = "local" ]; then - IMAGE="${NOMOS_TESTNET_IMAGE:-logos-blockchain-testing:local}" - export NOMOS_TESTNET_IMAGE_PULL_POLICY="${NOMOS_TESTNET_IMAGE_PULL_POLICY:-IfNotPresent}" - elif [ "${selection}" = "ecr" ]; then - local tag="${TAG:-test}" - if [ -n "${ECR_IMAGE:-}" ]; then - IMAGE="${ECR_IMAGE}" - elif [ -n "${ECR_REGISTRY:-}" ]; then - local registry="${ECR_REGISTRY}" - local repo="${ECR_REPO:-logos/logos-blockchain}" - IMAGE="${registry}/${repo}:${tag}" - elif [ -n "${AWS_ACCOUNT_ID:-}" ]; then - local aws_region="${AWS_REGION:-ap-southeast-2}" - local aws_account_id="${AWS_ACCOUNT_ID}" - local repo="${ECR_REPO:-logos-blockchain-testing}" - IMAGE="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com/${repo}:${tag}" - else - local registry="public.ecr.aws/r4s5t9y4" - local repo="${ECR_REPO:-logos/logos-blockchain}" - IMAGE="${registry}/${repo}:${tag}" - fi - export NOMOS_TESTNET_IMAGE_PULL_POLICY="${NOMOS_TESTNET_IMAGE_PULL_POLICY:-Always}" - else - fail_with_usage "Unknown image selection mode: ${selection}" - fi - - export IMAGE_TAG="${IMAGE}" - export NOMOS_TESTNET_IMAGE="${IMAGE}" - - if [ "${MODE}" = "k8s" ]; then - if [ "${selection}" = "ecr" ]; then - export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-inImage}" - else - export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-hostPath}" - fi - fi -} - -select_image - -if ! [[ "${RUN_SECS_RAW}" =~ ^[0-9]+$ ]] || [ "${RUN_SECS_RAW}" -le 0 ]; then - fail_with_usage "run-seconds must be a positive integer (pass -t/--run-seconds)" -fi -readonly RUN_SECS="${RUN_SECS_RAW}" -if [ -n "${DEMO_VALIDATORS}" ] && ! [[ "${DEMO_VALIDATORS}" =~ ^[0-9]+$ ]] ; then - fail_with_usage "validators must be a non-negative integer (pass -v/--validators)" -fi -if [ -n "${DEMO_EXECUTORS}" ] && ! [[ "${DEMO_EXECUTORS}" =~ ^[0-9]+$ ]] ; then - fail_with_usage "executors must be a non-negative integer (pass -e/--executors)" -fi -if [ -z "${DEMO_VALIDATORS}" ] || [ -z "${DEMO_EXECUTORS}" ]; then - fail_with_usage "validators and executors must be provided via -v/--validators and -e/--executors" -fi - -default_tar_path() { - # Pick a sensible default tarball based on mode and version. - if [ -n "${NOMOS_BINARIES_TAR:-}" ]; then - echo "${NOMOS_BINARIES_TAR}" - return - fi - case "$MODE" in - host) - echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" - ;; - compose|k8s) - # When skipping image rebuild, we need host-arch tools (witness generators) on the runner. - if [ "${NOMOS_SKIP_IMAGE_BUILD:-}" = "1" ]; then - echo "${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" - else - echo "${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz" - fi - ;; - *) echo "${ROOT_DIR}/.tmp/nomos-binaries-${VERSION}.tar.gz" ;; - esac -} - -restore_binaries_from_tar() { - local tar_path - if [ -n "${_RESTORE_TAR_OVERRIDE:-}" ]; then - tar_path="${_RESTORE_TAR_OVERRIDE}" - else - tar_path="$(default_tar_path)" - fi - if ! bundle_matches_expected "${tar_path}"; then - return 1 - fi - local extract_dir="${ROOT_DIR}/.tmp/nomos-binaries" - if [ ! -f "$tar_path" ]; then - return 1 - fi - echo "==> Restoring binaries from ${tar_path}" - rm -rf "${extract_dir}" - mkdir -p "${extract_dir}" - if ! tar -xzf "$tar_path" -C "${extract_dir}"; then - echo "Failed to extract ${tar_path}" >&2 - return 1 - fi - local src="${extract_dir}/artifacts" - local bin_dst="${ROOT_DIR}/testing-framework/assets/stack/bin" - local circuits_src="${src}/circuits" - local circuits_dst="${HOST_KZG_DIR}" - RESTORED_BIN_DIR="${src}" - export RESTORED_BIN_DIR - if [ -f "${src}/nomos-node" ] && [ -f "${src}/nomos-executor" ] && [ -f "${src}/nomos-cli" ]; then - local copy_bins=1 - if [ "$MODE" != "host" ] && ! host_bin_matches_arch "${src}/nomos-node"; then - echo "Bundled binaries do not match host arch; skipping copy so containers rebuild from source." - copy_bins=0 - rm -f "${bin_dst}/nomos-node" "${bin_dst}/nomos-executor" "${bin_dst}/nomos-cli" - fi - if [ "$copy_bins" -eq 1 ]; then - mkdir -p "${bin_dst}" - cp "${src}/nomos-node" "${src}/nomos-executor" "${src}/nomos-cli" "${bin_dst}/" - fi - else - echo "Binaries missing in ${tar_path}; provide a prebuilt binaries tarball." >&2 - return 1 - fi - if [ -d "${circuits_src}" ] && [ -f "${circuits_src}/${KZG_FILE}" ]; then - rm -rf "${circuits_dst}" - mkdir -p "${circuits_dst}" - if command -v rsync >/dev/null 2>&1; then - rsync -a --delete "${circuits_src}/" "${circuits_dst}/" - else - rm -rf "${circuits_dst:?}/"* - cp -a "${circuits_src}/." "${circuits_dst}/" - fi - else - echo "Circuits missing in ${tar_path}; provide a prebuilt binaries/circuits tarball." >&2 - return 1 - fi - RESTORED_BINARIES=1 - export RESTORED_BINARIES -} - -bundle_matches_expected() { - local tar_path="$1" - if [ ! -f "${tar_path}" ]; then - return 1 - fi - if [ -z "${NOMOS_NODE_REV:-}" ]; then - return 0 - fi - local meta - meta="$(tar -xOzf "${tar_path}" artifacts/nomos-bundle-meta.env 2>/dev/null || true)" - if [ -z "${meta}" ]; then - echo "Bundle meta missing in ${tar_path}; treating as stale and rebuilding." >&2 - return 1 - fi - local tar_rev tar_head - tar_rev="$(echo "${meta}" | sed -n 's/^nomos_node_rev=//p' | head -n 1)" - tar_head="$(echo "${meta}" | sed -n 's/^nomos_node_git_head=//p' | head -n 1)" - if [ -n "${tar_rev}" ] && [ "${tar_rev}" != "${NOMOS_NODE_REV}" ]; then - echo "Bundle ${tar_path} is for nomos-node rev ${tar_rev}, expected ${NOMOS_NODE_REV}; rebuilding." >&2 - return 1 - fi - if [ -n "${tar_head}" ] && [ "${tar_head}" != "${NOMOS_NODE_REV}" ]; then - echo "Bundle ${tar_path} is for nomos-node git head ${tar_head}, expected ${NOMOS_NODE_REV}; rebuilding." >&2 - return 1 - fi - return 0 -} - -ensure_binaries_tar() { - local platform="$1" - local tar_path="$2" - echo "==> Building fresh binaries bundle (${platform}) at ${tar_path}" - "${ROOT_DIR}/scripts/build-bundle.sh" --platform "${platform}" --output "${tar_path}" --rev "${NOMOS_NODE_REV}" -} - -host_bin_matches_arch() { - local bin_path="$1" - if [ ! -x "$bin_path" ]; then - return 1 - fi - local info expected - info="$(file -b "$bin_path" 2>/dev/null || true)" - case "$(uname -m)" in - x86_64) expected="x86-64|x86_64" ;; - aarch64|arm64) expected="arm64|aarch64" ;; - *) expected="" ;; - esac - if [ -n "$expected" ] && echo "$info" | grep -Eqi "$expected"; then - return 0 - fi - return 1 -} - -HOST_TAR="${ROOT_DIR}/.tmp/nomos-binaries-host-${VERSION}.tar.gz" -LINUX_TAR="${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz" -NEED_HOST_RESTORE_AFTER_IMAGE=0 - -if [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN}" ]; then - echo "==> Using pre-specified host binaries (NOMOS_NODE_BIN/NOMOS_EXECUTOR_BIN); skipping tarball restore" -else - # On non-Linux compose/k8s runs, use the Linux bundle for image build, then restore host bundle for the runner. - if [ "$MODE" != "host" ] && [ "$(uname -s)" != "Linux" ] && [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "0" ] && [ -f "${LINUX_TAR}" ]; then - NEED_HOST_RESTORE_AFTER_IMAGE=1 - _RESTORE_TAR_OVERRIDE="${LINUX_TAR}" restore_binaries_from_tar || { - ensure_binaries_tar linux "${LINUX_TAR}" - _RESTORE_TAR_OVERRIDE="${LINUX_TAR}" restore_binaries_from_tar - } - unset _RESTORE_TAR_OVERRIDE - fi - - if ! restore_binaries_from_tar; then - tar_path="$(default_tar_path)" - case "$MODE" in - host) ensure_binaries_tar host "${tar_path}" ;; - compose|k8s) - if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then - ensure_binaries_tar host "${tar_path}" - else - ensure_binaries_tar linux "${tar_path}" - fi - ;; - *) ensure_binaries_tar host "${tar_path}" ;; - esac - restore_binaries_from_tar || { - echo "ERROR: Missing or invalid binaries tarball. Provide it via NOMOS_BINARIES_TAR or place it at $(default_tar_path)." >&2 - exit 1 - } - fi -fi - -echo "==> Using restored circuits/binaries bundle" -SETUP_OUT="$(mktemp -t nomos-setup-output.XXXXXX)" -if [ "$MODE" != "host" ]; then - if [ "${NOMOS_SKIP_IMAGE_BUILD:-0}" = "1" ]; then - echo "==> Skipping testnet image rebuild (NOMOS_SKIP_IMAGE_BUILD=1)" - else - echo "==> Rebuilding testnet image (${IMAGE})" - IMAGE_TAG="${IMAGE}" COMPOSE_CIRCUITS_PLATFORM="${COMPOSE_CIRCUITS_PLATFORM:-}" \ - "${ROOT_DIR}/testing-framework/assets/stack/scripts/build_test_image.sh" - fi -fi - -if [ "${NEED_HOST_RESTORE_AFTER_IMAGE}" = "1" ]; then - if [ -f "${HOST_TAR}" ]; then - echo "==> Restoring host bundle for runner (${HOST_TAR})" - _RESTORE_TAR_OVERRIDE="${HOST_TAR}" restore_binaries_from_tar || { - ensure_binaries_tar host "${HOST_TAR}" - _RESTORE_TAR_OVERRIDE="${HOST_TAR}" restore_binaries_from_tar || { - echo "ERROR: Failed to restore host bundle from ${HOST_TAR}" >&2 - exit 1 - } - } - unset _RESTORE_TAR_OVERRIDE - echo "==> Using restored circuits/binaries bundle" - else - ensure_binaries_tar host "${HOST_TAR}" - _RESTORE_TAR_OVERRIDE="${HOST_TAR}" restore_binaries_from_tar || { - echo "ERROR: Failed to restore host bundle from ${HOST_TAR}" >&2 - exit 1 - } - unset _RESTORE_TAR_OVERRIDE - echo "==> Using restored circuits/binaries bundle" - fi -fi - -HOST_BUNDLE_PATH="${HOST_KZG_DIR}" - -# If the host bundle was somehow pruned, repair it once more. -if [ ! -x "${HOST_BUNDLE_PATH}/zksign/witness_generator" ]; then - echo "ERROR: Missing zksign/witness_generator in restored bundle; ensure the tarball contains host-compatible circuits." >&2 - exit 1 -fi -KZG_HOST_PATH="${HOST_BUNDLE_PATH}/${KZG_FILE}" -if [ ! -f "${KZG_HOST_PATH}" ]; then - echo "ERROR: KZG params missing at ${KZG_HOST_PATH}; ensure the tarball contains circuits." >&2 - exit 1 -fi - -if [ "$MODE" = "host" ]; then - if [ -n "${NOMOS_NODE_BIN:-}" ] && [ -x "${NOMOS_NODE_BIN}" ] && [ -n "${NOMOS_EXECUTOR_BIN:-}" ] && [ -x "${NOMOS_EXECUTOR_BIN}" ]; then - echo "==> Using provided host binaries (env override)" - else - tar_node="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-node" - tar_exec="${RESTORED_BIN_DIR:-${ROOT_DIR}/testing-framework/assets/stack/bin}/nomos-executor" - if [ ! -x "${tar_node}" ] || [ ! -x "${tar_exec}" ]; then - echo "ERROR: Restored tarball missing host executables; provide a host-compatible binaries tarball." >&2 - exit 1 - fi - if ! host_bin_matches_arch "${tar_node}" || ! host_bin_matches_arch "${tar_exec}"; then - echo "ERROR: Restored executables do not match host architecture; provide a host-compatible binaries tarball." >&2 - exit 1 - fi - echo "==> Using restored host binaries from tarball" - NOMOS_NODE_BIN="${tar_node}" - NOMOS_EXECUTOR_BIN="${tar_exec}" - export NOMOS_NODE_BIN NOMOS_EXECUTOR_BIN - fi -fi - -echo "==> Running ${BIN} for ${RUN_SECS}s" -cd "${ROOT_DIR}" -if [ "$MODE" = "compose" ] || [ "$MODE" = "k8s" ]; then - if [ "$MODE" = "k8s" ] && [ "${NOMOS_KZG_MODE:-hostPath}" = "inImage" ]; then - KZG_PATH="${NOMOS_KZG_IN_IMAGE_PARAMS_PATH:-/opt/nomos/kzg-params/kzgrs_test_params}" - else - KZG_PATH="${KZG_CONTAINER_PATH}" - fi -else - KZG_PATH="${KZG_HOST_PATH}" -fi - -# Ensure compose image pulls circuits for the host architecture by default. -if [ "$MODE" = "compose" ] && [ -z "${COMPOSE_CIRCUITS_PLATFORM:-}" ]; then - arch="$(uname -m)" - case "$arch" in - x86_64) COMPOSE_CIRCUITS_PLATFORM="linux-x86_64" ;; - arm64|aarch64) COMPOSE_CIRCUITS_PLATFORM="linux-aarch64" ;; - *) COMPOSE_CIRCUITS_PLATFORM="linux-x86_64" ;; - esac -fi - -export NOMOS_DEMO_RUN_SECS="${RUN_SECS}" - -if [ -n "${DEMO_VALIDATORS}" ]; then - export NOMOS_DEMO_VALIDATORS="${DEMO_VALIDATORS}" -fi -if [ -n "${DEMO_EXECUTORS}" ]; then - export NOMOS_DEMO_EXECUTORS="${DEMO_EXECUTORS}" -fi -POL_PROOF_DEV_MODE=true \ -TESTNET_PRINT_ENDPOINTS=1 \ -NOMOS_TESTNET_IMAGE="${IMAGE}" \ -NOMOS_CIRCUITS="${HOST_BUNDLE_PATH}" \ -NOMOS_KZGRS_PARAMS_PATH="${KZG_PATH}" \ -NOMOS_NODE_BIN="${NOMOS_NODE_BIN:-}" \ -NOMOS_EXECUTOR_BIN="${NOMOS_EXECUTOR_BIN:-}" \ -COMPOSE_CIRCUITS_PLATFORM="${COMPOSE_CIRCUITS_PLATFORM:-}" \ - cargo run -p runner-examples --bin "${BIN}" +run_examples::main "$@" diff --git a/scripts/setup-circuits-stack.sh b/scripts/setup-circuits-stack.sh index fc47bc9..f87635e 100755 --- a/scripts/setup-circuits-stack.sh +++ b/scripts/setup-circuits-stack.sh @@ -1,136 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -# One-stop helper to prepare circuits for both the Docker image (Linux/x86_64) -# and the host (for witness generators). It populates -# testing-framework/assets/stack/kzgrs_test_params with a Linux bundle for the -# image, and if the host is not Linux/x86_64, it also fetches a host-native -# bundle and tells you where to point NOMOS_CIRCUITS. -# -# Usage: scripts/setup-circuits-stack.sh [VERSION] -# VERSION defaults to v0.3.1 -# -# Env overrides: -# STACK_DIR - where to place the Linux bundle (default: testing-framework/assets/stack/kzgrs_test_params) -# HOST_DIR - where to place the host bundle (default: .tmp/nomos-circuits-host) -# NOMOS_CIRCUITS_PLATFORM - force host platform (e.g., macos-aarch64) -# NOMOS_CIRCUITS_REBUILD_RAPIDSNARK - set to 1 to force rebuild (not needed for mac arm/x86 bundles) +# Thin wrapper; the actual implementation lives in scripts/lib/setup-circuits-stack.sh +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [ -f "${ROOT_DIR}/versions.env" ]; then - # shellcheck disable=SC1091 - . "${ROOT_DIR}/versions.env" -fi -if [ -f "${ROOT_DIR}/paths.env" ]; then - # shellcheck disable=SC1091 - . "${ROOT_DIR}/paths.env" -fi -KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}" -KZG_FILE="${NOMOS_KZG_FILE:-kzgrs_test_params}" -HOST_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_HOST_DIR_REL:-.tmp/nomos-circuits-host}" -LINUX_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_LINUX_DIR_REL:-.tmp/nomos-circuits-linux}" -LINUX_STAGE_DIR="${LINUX_STAGE_DIR:-${ROOT_DIR}/${LINUX_DIR_REL_DEFAULT}}" -HOST_DIR_REL_DEFAULT="${NOMOS_CIRCUITS_HOST_DIR_REL:-.tmp/nomos-circuits-host}" -VERSION="${1:-${VERSION:-v0.3.1}}" -STACK_DIR="${STACK_DIR:-${ROOT_DIR}/${KZG_DIR_REL}}" -HOST_DIR="${HOST_DIR:-${ROOT_DIR}/${HOST_DIR_REL_DEFAULT}}" -NOMOS_NODE_REV="${NOMOS_NODE_REV:-d2dd5a5084e1daef4032562c77d41de5e4d495f8}" -# Force non-interactive installs so repeated runs do not prompt. -export NOMOS_CIRCUITS_NONINTERACTIVE=1 +# shellcheck disable=SC1091 +. "${ROOT_DIR}/scripts/lib/setup-circuits-stack.sh" -detect_platform() { - local os arch - case "$(uname -s)" in - Linux*) os="linux" ;; - Darwin*) os="macos" ;; - MINGW*|MSYS*|CYGWIN*) os="windows" ;; - *) echo "Unsupported OS: $(uname -s)" >&2; exit 1 ;; - esac - - case "$(uname -m)" in - x86_64) arch="x86_64" ;; - aarch64|arm64) arch="aarch64" ;; - *) echo "Unsupported arch: $(uname -m)" >&2; exit 1 ;; - esac - - echo "${os}-${arch}" -} - -fetch_bundle() { - local platform="$1" - local dest="$2" - local rebuild="${3:-0}" - - rm -rf "$dest" - mkdir -p "$dest" - - NOMOS_CIRCUITS_PLATFORM="$platform" \ - NOMOS_CIRCUITS_REBUILD_RAPIDSNARK="$rebuild" \ - "${ROOT_DIR}/scripts/setup-nomos-circuits.sh" "$VERSION" "$dest" -} - -fetch_kzg_params() { - local dest_dir="$1" - local dest_file="${dest_dir}/${KZG_FILE}" - local url="https://raw.githubusercontent.com/logos-co/nomos-node/${NOMOS_NODE_REV}/tests/kzgrs/kzgrs_test_params" - - echo "Fetching KZG parameters from ${url}" - curl -fsSL "$url" -o "$dest_file" -} - -echo "Preparing circuits (version ${VERSION})" -echo "Workspace: ${ROOT_DIR}" - -LINUX_PLATFORM="linux-x86_64" - -echo "Installing Linux bundle for Docker image into ${STACK_DIR}" -stage_real="$(python3 - "${LINUX_STAGE_DIR}" <<'PY' -import os, sys -print(os.path.realpath(sys.argv[1])) -PY -)" -stack_real="$(python3 - "${STACK_DIR}" <<'PY' -import os, sys -print(os.path.realpath(sys.argv[1])) -PY -)" - -if [ "$stage_real" = "$stack_real" ]; then - # No staging copy needed; install directly into STACK_DIR. - rm -rf "$STACK_DIR" - fetch_bundle "$LINUX_PLATFORM" "$STACK_DIR" 0 - fetch_kzg_params "$STACK_DIR" -else - rm -rf "${LINUX_STAGE_DIR}" - mkdir -p "${LINUX_STAGE_DIR}" - fetch_bundle "$LINUX_PLATFORM" "${LINUX_STAGE_DIR}" 0 - rm -rf "$STACK_DIR" - mkdir -p "$STACK_DIR" - cp -R "${LINUX_STAGE_DIR}/." "$STACK_DIR/" - fetch_kzg_params "$STACK_DIR" -fi -echo "Linux bundle ready at ${STACK_DIR}" - -host_platform="${NOMOS_CIRCUITS_PLATFORM:-$(detect_platform)}" -if [[ "$host_platform" == "$LINUX_PLATFORM" ]]; then - echo "Host platform ${host_platform} matches Linux bundle; host can reuse ${STACK_DIR}" - echo "Export if you want to be explicit:" - echo " export NOMOS_CIRCUITS=\"${STACK_DIR}\"" -else - echo "Host platform detected: ${host_platform}; installing host-native bundle into ${HOST_DIR}" - fetch_bundle "$host_platform" "$HOST_DIR" "${NOMOS_CIRCUITS_REBUILD_RAPIDSNARK:-0}" - fetch_kzg_params "$HOST_DIR" - echo "Host bundle ready at ${HOST_DIR}" - echo - echo "Set for host runs:" - echo " export NOMOS_CIRCUITS=\"${HOST_DIR}\"" -fi - -cat <<'EOF' - -Done. -- For Docker/compose: rebuild the image to bake the Linux bundle: - testing-framework/assets/stack/scripts/build_test_image.sh -- For host runs (e.g., compose_runner): ensure NOMOS_CIRCUITS points to the host bundle above. -EOF +setup_circuits_stack::main "$@" diff --git a/scripts/setup-nomos-circuits.sh b/scripts/setup-nomos-circuits.sh index 2d16db9..a41ee07 100755 --- a/scripts/setup-nomos-circuits.sh +++ b/scripts/setup-nomos-circuits.sh @@ -1,250 +1,15 @@ -#!/bin/bash -# -# Setup script for nomos-circuits -# -# Usage: ./setup-nomos-circuits.sh [VERSION] [INSTALL_DIR] -# -# Arguments: -# VERSION - Optional. Version to install (default: v0.3.1) -# INSTALL_DIR - Optional. Installation directory (default: $HOME/.nomos-circuits) -# -# Examples: -# ./setup-nomos-circuits.sh # Install default version to default location -# ./setup-nomos-circuits.sh v0.2.0 # Install specific version to default location -# ./setup-nomos-circuits.sh v0.2.0 /opt/circuits # Install to custom location +#!/usr/bin/env bash +set -euo pipefail -set -e +# Thin wrapper; the actual implementation lives in scripts/lib/setup-nomos-circuits.sh +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi -# Default values -VERSION="${1:-v0.3.1}" -DEFAULT_INSTALL_DIR="$HOME/.nomos-circuits" -INSTALL_DIR="${2:-$DEFAULT_INSTALL_DIR}" -REPO="logos-co/nomos-circuits" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -NONINTERACTIVE="${NOMOS_CIRCUITS_NONINTERACTIVE:-0}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color +# shellcheck disable=SC1091 +. "${ROOT_DIR}/scripts/lib/setup-nomos-circuits.sh" -print_info() { - echo -e "${BLUE}ℹ${NC} $1" -} +setup_nomos_circuits::main "$@" -print_success() { - echo -e "${GREEN}✓${NC} $1" -} - -print_warning() { - echo -e "${YELLOW}⚠${NC} $1" -} - -print_error() { - echo -e "${RED}✗${NC} $1" -} - -# Detect OS and architecture -detect_platform() { - local os="" - local arch="" - - # Detect OS - case "$(uname -s)" in - Linux*) os="linux";; - Darwin*) os="macos";; - MINGW*|MSYS*|CYGWIN*) os="windows";; - *) print_error "Unsupported operating system: $(uname -s)"; exit 1;; - esac - - # Detect architecture - case "$(uname -m)" in - x86_64) arch="x86_64";; - aarch64) arch="aarch64";; - arm64) arch="aarch64";; - *) print_error "Unsupported architecture: $(uname -m)"; exit 1;; - esac - - echo "${os}-${arch}" -} - -# Check if installation directory exists and get confirmation -check_existing_installation() { - if [ -d "$INSTALL_DIR" ]; then - print_warning "Installation directory already exists: $INSTALL_DIR" - - # Check if it has a VERSION file - if [ -f "$INSTALL_DIR/VERSION" ]; then - local current_version=$(cat "$INSTALL_DIR/VERSION") - print_info "Currently installed version: $current_version" - fi - - # In non-interactive environments (CI), automatically overwrite - if [ "$NONINTERACTIVE" = "1" ] || [ ! -t 0 ]; then - print_info "Non-interactive environment detected, automatically overwriting..." - else - # Interactive environment - ask for confirmation - echo - read -p "Do you want to overwrite it? (y/N): " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - print_info "Installation cancelled." - exit 0 - fi - fi - - print_info "Removing existing installation..." - rm -rf "$INSTALL_DIR" - fi -} - -# Download and extract the release -download_release() { - local platform="$1" - local artifact="nomos-circuits-${VERSION}-${platform}.tar.gz" - local url="https://github.com/${REPO}/releases/download/${VERSION}/${artifact}" - local temp_dir=$(mktemp -d) - - print_info "Downloading nomos-circuits ${VERSION} for ${platform}..." - print_info "URL: $url" - - # Build curl command with optional authentication - local curl_cmd="curl -fL --retry 5 --retry-delay 2 --retry-all-errors" - if [ -n "$GITHUB_TOKEN" ]; then - curl_cmd="$curl_cmd --header 'authorization: Bearer ${GITHUB_TOKEN}'" - fi - curl_cmd="$curl_cmd -o ${temp_dir}/${artifact} $url" - - if ! eval "$curl_cmd"; then - print_error "Failed to download release artifact" - print_error "Please check that version ${VERSION} exists for platform ${platform}" - print_error "Available releases: https://github.com/${REPO}/releases" - rm -rf "$temp_dir" - return 1 - fi - - print_success "Download complete" - - # Validate archive before extracting - if ! tar -tzf "${temp_dir}/${artifact}" >/dev/null 2>&1; then - print_error "Downloaded archive is not a valid tar.gz: ${temp_dir}/${artifact}" - rm -rf "$temp_dir" - return 1 - fi - - print_info "Extracting to ${INSTALL_DIR}..." - mkdir -p "$INSTALL_DIR" - - if ! tar -xzf "${temp_dir}/${artifact}" -C "$INSTALL_DIR" --strip-components=1; then - print_error "Failed to extract archive" - rm -rf "$temp_dir" - return 1 - fi - - rm -rf "$temp_dir" - print_success "Extraction complete" -} - -# Handle macOS code signing/quarantine issues -handle_macos_quarantine() { - print_info "macOS detected: Removing quarantine attributes from executables..." - - # Remove quarantine attribute from all executable files - if find "$INSTALL_DIR" -type f -perm +111 -exec xattr -d com.apple.quarantine {} \; 2>/dev/null; then - print_success "Quarantine attributes removed" - else - print_warning "Could not remove quarantine attributes (they may not exist)" - fi -} - -# Main installation process -main() { - print_info "Setting up nomos-circuits ${VERSION}" - print_info "Installation directory: $INSTALL_DIR" - echo - - # Detect platform (allow override via NOMOS_CIRCUITS_PLATFORM) - local platform_override="${NOMOS_CIRCUITS_PLATFORM:-}" - local platform - if [ -n "$platform_override" ]; then - platform="$platform_override" - print_info "Using overridden platform: $platform" - else - platform=$(detect_platform) - print_info "Detected platform: $platform" - fi - - # Check existing installation - check_existing_installation - - local rebuild_required="${NOMOS_CIRCUITS_REBUILD_RAPIDSNARK:-0}" - - # Download and extract (with fallbacks) - if ! download_release "$platform"; then - if [[ "$platform" == linux-aarch64 ]]; then - print_warning "Falling back to linux-x86_64 circuits bundle; will rebuild prover for aarch64." - rm -rf "$INSTALL_DIR" - if ! download_release "linux-x86_64"; then - exit 1 - fi - rebuild_required=1 - elif [[ "$platform" == macos-x86_64 ]]; then - print_warning "No macOS x86_64 bundle; falling back to macOS aarch64 circuits bundle and rebuilding prover." - rm -rf "$INSTALL_DIR" - if ! download_release "macos-aarch64"; then - print_warning "macOS aarch64 bundle unavailable; trying linux-x86_64 bundle and rebuilding prover." - rm -rf "$INSTALL_DIR" - if ! download_release "linux-x86_64"; then - exit 1 - fi - fi - rebuild_required=1 - else - exit 1 - fi - fi - - # Handle macOS quarantine if needed - if [[ "$platform" == macos-* ]]; then - echo - handle_macos_quarantine - fi - - if [[ "$rebuild_required" == "1" ]]; then - echo - print_info "Rebuilding rapidsnark prover for ${platform}..." - "${SCRIPT_DIR}/build-rapidsnark.sh" "$INSTALL_DIR" - else - print_info "Skipping rapidsnark rebuild (set NOMOS_CIRCUITS_REBUILD_RAPIDSNARK=1 to force)." - fi - - echo - print_success "Installation complete!" - echo - print_info "nomos-circuits ${VERSION} is now installed at: $INSTALL_DIR" - print_info "The following circuits are available:" - - # Discover circuits by finding directories that contain a witness_generator - for dir in "$INSTALL_DIR"/*/; do - if [ -d "$dir" ]; then - local circuit_name - circuit_name=$(basename "$dir") - if [ -f "$dir/witness_generator" ]; then - echo " • $circuit_name" - fi - fi - done - - # Only show export instructions if not using the default location - if [ "$INSTALL_DIR" != "$DEFAULT_INSTALL_DIR" ]; then - echo - print_info "Since you're using a custom installation directory, set the environment variable:" - print_info " export NOMOS_CIRCUITS=$INSTALL_DIR" - echo - fi -} - -# Run main -main diff --git a/scripts/update-grafana-panel-titles.py b/scripts/update-grafana-panel-titles.py new file mode 100644 index 0000000..becce1b --- /dev/null +++ b/scripts/update-grafana-panel-titles.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +import glob +import json +import os +import re +from typing import Any, Optional + + +DASH_DIR = "testing-framework/assets/stack/monitoring/grafana/dashboards" + +TITLE_SEP = " — " + + +METRIC_RE = re.compile(r"\b[a-zA-Z_:][a-zA-Z0-9_:]*\b") + + +def _collect_exprs(panel: dict[str, Any]) -> list[str]: + exprs: list[str] = [] + for target in panel.get("targets") or []: + expr = target.get("expr") + if isinstance(expr, str) and expr.strip(): + exprs.append(expr.strip()) + return exprs + + +def _descriptor_from_exprs(title: str, exprs: list[str]) -> Optional[str]: + if not exprs: + return None + + all_expr = "\n".join(exprs) + + if "histogram_quantile" in all_expr: + return "p95 latency" + + if "time() - on() (" in all_expr: + return "time since last" + + if "consensus_tip_height - consensus_finalized_height" in all_expr: + return "finalization gap" + + if any("rate(" in e for e in exprs) or any("irate(" in e for e in exprs): + return "events/sec" + + lower_title = title.lower() + if "throughput" in lower_title or "tps" in lower_title: + return "tx/sec" + + if "errors" in lower_title or "fail" in lower_title: + return "error rate" + + if "peers" in lower_title: + return "peer count" + + if "connections" in lower_title: + return "conn count" + + if "queue" in lower_title or "pending" in lower_title: + return "queue depth" + + # If the title didn't help, infer from obvious metric names. + metrics = {m for m in METRIC_RE.findall(all_expr) if "_" in m or ":" in m} + if any(m.endswith("_pending") for m in metrics): + return "queue depth" + if any(m.endswith("_height") for m in metrics): + return "height" + if any(m.endswith("_slot") for m in metrics): + return "slot" + if any(m.endswith("_epoch") for m in metrics): + return "epoch" + if any("connections" in m for m in metrics): + return "conn count" + + return "current" + + +def _update_panel_title(panel: dict[str, Any]) -> bool: + if panel.get("type") == "row": + return False + + title = panel.get("title") + if not isinstance(title, str) or not title.strip(): + return False + + if TITLE_SEP in title: + return False + + exprs = _collect_exprs(panel) + desc = _descriptor_from_exprs(title, exprs) + if not desc: + return False + + panel["title"] = f"{title}{TITLE_SEP}{desc}" + return True + + +def main() -> int: + paths = sorted(glob.glob(os.path.join(DASH_DIR, "*.json"))) + if not paths: + raise SystemExit(f"No dashboards found at {DASH_DIR}") + + changed_files = 0 + changed_panels = 0 + + for path in paths: + with open(path) as f: + dash = json.load(f) + + changed = False + for panel in dash.get("panels") or []: + if _update_panel_title(panel): + changed = True + changed_panels += 1 + + if changed: + with open(path, "w") as f: + json.dump(dash, f, indent=2, sort_keys=False) + f.write("\n") + changed_files += 1 + + print(f"updated {changed_panels} panels across {changed_files} dashboards") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + diff --git a/scripts/update-nomos-rev.sh b/scripts/update-nomos-rev.sh index e697725..30bef5b 100755 --- a/scripts/update-nomos-rev.sh +++ b/scripts/update-nomos-rev.sh @@ -1,211 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -# Update nomos-node source across versions.env and Cargo.toml. -# Usage: -# scripts/update-nomos-rev.sh --rev -# scripts/update-nomos-rev.sh --path -# -# Only one of --rev/--path may be supplied. - -usage() { - cat <<'EOF' -Usage: - scripts/update-nomos-rev.sh --rev - scripts/update-nomos-rev.sh --path - scripts/update-nomos-rev.sh --unskip-worktree - -Notes: - --rev sets NOMOS_NODE_REV and updates Cargo.toml revs - --path sets NOMOS_NODE_PATH (clears NOMOS_NODE_REV) and patches Cargo.toml to use a local nomos-node checkout - --unskip-worktree clears any skip-worktree flag for Cargo.toml - Only one may be used at a time. -EOF -} - -REV="" -LOCAL_PATH="" -UNSKIP_WORKTREE=0 -while [ "$#" -gt 0 ]; do - case "$1" in - --rev) REV="${2:-}"; shift 2 ;; - --path) LOCAL_PATH="${2:-}"; shift 2 ;; - --unskip-worktree) UNSKIP_WORKTREE=1; shift ;; - -h|--help) usage; exit 0 ;; - *) echo "Unknown arg: $1" >&2; usage; exit 1 ;; - esac -done - -if [ "${UNSKIP_WORKTREE}" -eq 1 ] && { [ -n "${REV}" ] || [ -n "${LOCAL_PATH}" ]; }; then - echo "Use --unskip-worktree alone." >&2 - usage - exit 1 -fi - -if [ -n "${REV}" ] && [ -n "${LOCAL_PATH}" ]; then - echo "Use either --rev or --path, not both" >&2 - usage; exit 1 -fi - -if [ -z "${REV}" ] && [ -z "${LOCAL_PATH}" ] && [ "${UNSKIP_WORKTREE}" -eq 0 ]; then - usage; exit 1 +# Thin wrapper; the actual implementation lives in scripts/lib/update-nomos-rev.sh +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" fi ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -if [ ! -f "${ROOT_DIR}/versions.env" ]; then - echo "ERROR: versions.env missing; run from repo root." >&2 - exit 1 -fi +# shellcheck disable=SC1091 +. "${ROOT_DIR}/scripts/lib/update-nomos-rev.sh" -# Ensure keys exist so sed replacements succeed even if they were absent. -ensure_env_key() { - local key="$1" default_value="$2" - if ! grep -Eq "^#?[[:space:]]*${key}=" "${ROOT_DIR}/versions.env"; then - echo "${default_value}" >> "${ROOT_DIR}/versions.env" - fi -} -ensure_env_key "NOMOS_NODE_REV" "# NOMOS_NODE_REV=" -ensure_env_key "NOMOS_NODE_PATH" "# NOMOS_NODE_PATH=" - -maybe_unskip_worktree() { - local file="$1" - if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - git -C "${ROOT_DIR}" update-index --no-skip-worktree "${file}" >/dev/null 2>&1 || true - fi -} - -maybe_skip_worktree() { - local file="$1" - if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - git -C "${ROOT_DIR}" update-index --skip-worktree "${file}" >/dev/null 2>&1 || true - fi -} - -if [ "${UNSKIP_WORKTREE}" -eq 1 ]; then - maybe_unskip_worktree "Cargo.toml" - echo "Cleared skip-worktree on Cargo.toml (if it was set)." - exit 0 -fi - -if [ -n "${REV}" ]; then - echo "Updating nomos-node rev to ${REV}" - # Update versions.env NOMOS_NODE_REV entry, clear NOMOS_NODE_PATH if present. - sed -i.bak -E \ - -e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/NOMOS_NODE_REV=${REV}/" \ - -e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/# NOMOS_NODE_PATH=/" \ - "${ROOT_DIR}/versions.env" - rm -f "${ROOT_DIR}/versions.env.bak" - - # Update all rev fields in Cargo.toml pointing to nomos-node. - python3 - "$ROOT_DIR" "$REV" <<'PY' -import pathlib, re, sys -root = pathlib.Path(sys.argv[1]) -rev = sys.argv[2] -cargo_toml = root / "Cargo.toml" -txt = cargo_toml.read_text() -txt = txt.replace("\\n", "\n") -txt = re.sub( - r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)', - "", - txt, -) -txt = re.sub( - r'(git = "https://github\.com/logos-co/nomos-node\.git", rev = ")[^"]+(")', - r"\g<1>" + rev + r"\2", - txt, -) -cargo_toml.write_text(txt.rstrip() + "\n") -PY - maybe_unskip_worktree "Cargo.toml" -else - echo "Pointing to local nomos-node at ${LOCAL_PATH}" - if [ ! -d "${LOCAL_PATH}" ]; then - echo "ERROR: path does not exist: ${LOCAL_PATH}" >&2 - exit 1 - fi - CURRENT_REV="$(grep -E '^[#[:space:]]*NOMOS_NODE_REV=' "${ROOT_DIR}/versions.env" | head -n1 | sed -E 's/^#?[[:space:]]*NOMOS_NODE_REV=//')" - # Update versions.env to favor the local path. - sed -i.bak -E \ - -e "s/^#?[[:space:]]*NOMOS_NODE_PATH=.*/NOMOS_NODE_PATH=${LOCAL_PATH//\//\\/}/" \ - -e "s/^#?[[:space:]]*NOMOS_NODE_REV=.*/# NOMOS_NODE_REV=${CURRENT_REV}/" \ - "${ROOT_DIR}/versions.env" - rm -f "${ROOT_DIR}/versions.env.bak" - - # Generate a patch block for all nomos-node crates we depend on. - PYTHON_BIN="${PYTHON_BIN:-python3}" - if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then - echo "ERROR: python3 is required to patch Cargo.toml for local paths" >&2 - exit 1 - fi - "${PYTHON_BIN}" - "$ROOT_DIR" "$LOCAL_PATH" <<'PY' -import json -import pathlib -import re -import subprocess -import sys - -root = pathlib.Path(sys.argv[1]) -node_path = pathlib.Path(sys.argv[2]) - -targets = [ - "broadcast-service", "chain-leader", "chain-network", "chain-service", - "common-http-client", "cryptarchia-engine", "cryptarchia-sync", - "executor-http-client", "groth16", "key-management-system-service", - "kzgrs", "kzgrs-backend", "nomos-api", "nomos-blend-message", - "nomos-blend-service", "nomos-core", "nomos-da-dispersal", - "nomos-da-network-core", "nomos-da-network-service", "nomos-da-sampling", - "nomos-da-verifier", "nomos-executor", "nomos-http-api-common", - "nomos-ledger", "nomos-libp2p", "nomos-network", "nomos-node", - "nomos-sdp", "nomos-time", "nomos-tracing", "nomos-tracing-service", - "nomos-utils", "nomos-wallet", "poc", "pol", "subnetworks-assignations", - "tests", "tx-service", "wallet", "zksign", -] - -try: - meta = subprocess.check_output( - ["cargo", "metadata", "--format-version", "1", "--no-deps"], - cwd=node_path, - ) -except subprocess.CalledProcessError as exc: - sys.stderr.write(f"Failed to run cargo metadata in {node_path}: {exc}\n") - sys.exit(1) - -data = json.loads(meta) -paths = {} -for pkg in data.get("packages", []): - paths[pkg["name"]] = str(pathlib.Path(pkg["manifest_path"]).parent) - -patch_lines = ['[patch."https://github.com/logos-co/nomos-node"]'] -missing = [] -for name in targets: - if name in paths: - patch_lines.append(f'{name} = {{ path = "{paths[name]}" }}') - else: - missing.append(name) - -cargo_toml = root / "Cargo.toml" -txt = cargo_toml.read_text() -txt = txt.replace("\\n", "\n") -txt = re.sub( - r'(?ms)^\[patch\."https://github\.com/logos-co/nomos-node"\].*?(?=^\[|\Z)', - "", - txt, -) -txt = txt.rstrip() + "\n\n" + "\n".join(patch_lines) + "\n" -cargo_toml.write_text(txt) - -if missing: - sys.stderr.write( - "Warning: missing crates in local nomos-node checkout: " - + ", ".join(missing) - + "\n" - ) -PY - - # Avoid accidentally committing the local patch block. - maybe_skip_worktree "Cargo.toml" - echo "Local nomos-node patch applied; Cargo.toml marked skip-worktree (run --unskip-worktree to clear)." -fi - -echo "Done. Consider updating Cargo.lock if needed (cargo fetch)." +update_nomos_rev::main "$@" diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/api-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/api-dashboard.json index d677bb3..03f63e5 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/api-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/api-dashboard.json @@ -56,7 +56,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -68,7 +70,7 @@ "refId": "A" } ], - "title": "HTTP Request Rate", + "title": "HTTP Request Rate \u2014 events/sec", "type": "timeseries" }, { @@ -106,7 +108,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -118,7 +122,7 @@ "refId": "A" } ], - "title": "Total HTTP Requests", + "title": "Total HTTP Requests \u2014 current", "type": "stat" }, { @@ -199,7 +203,7 @@ "refId": "B" } ], - "title": "HTTP Request Duration", + "title": "HTTP Request Duration \u2014 p95 latency", "type": "timeseries" }, { @@ -275,7 +279,7 @@ "refId": "A" } ], - "title": "HTTP Error Rate", + "title": "HTTP Error Rate \u2014 events/sec", "type": "timeseries" }, { @@ -361,7 +365,7 @@ "refId": "C" } ], - "title": "HTTP Status Codes", + "title": "HTTP Status Codes \u2014 events/sec", "type": "timeseries" }, { @@ -437,13 +441,17 @@ "refId": "A" } ], - "title": "Top API Endpoints", + "title": "Top API Endpoints \u2014 events/sec", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "api", "http"], + "tags": [ + "nomos", + "api", + "http" + ], "templating": { "list": [] }, @@ -457,4 +465,4 @@ "uid": "api-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/api-detailed-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/api-detailed-dashboard.json index 1424574..589c033 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/api-detailed-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/api-detailed-dashboard.json @@ -94,7 +94,7 @@ "refId": "A" } ], - "title": "HTTP Request Rates by Endpoint", + "title": "HTTP Request Rates by Endpoint \u2014 events/sec", "type": "timeseries" }, { @@ -141,7 +141,7 @@ } }, "mappings": [], - "unit": "ms" + "unit": "s" } }, "gridPos": { @@ -165,12 +165,12 @@ }, "targets": [ { - "expr": "http_request_duration_ms", - "legendFormat": "{{method}} {{endpoint}} - {{job}}", + "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))", + "legendFormat": "p95 {{method}} {{endpoint}} - {{job}}", "refId": "A" } ], - "title": "HTTP Request Duration by Endpoint", + "title": "HTTP Request Duration by Endpoint \u2014 p95 latency", "type": "timeseries" }, { @@ -208,7 +208,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -220,7 +222,7 @@ "refId": "A" } ], - "title": "Total Requests by Endpoint", + "title": "Total Requests by Endpoint \u2014 current", "type": "stat" }, { @@ -296,7 +298,7 @@ "refId": "A" } ], - "title": "Request Rate by HTTP Method", + "title": "Request Rate by HTTP Method \u2014 events/sec", "type": "timeseries" }, { @@ -372,13 +374,18 @@ "refId": "A" } ], - "title": "Top 10 Endpoints by Request Rate", + "title": "Top 10 Endpoints by Request Rate \u2014 events/sec", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "api", "http", "endpoints"], + "tags": [ + "nomos", + "api", + "http", + "endpoints" + ], "templating": { "list": [] }, @@ -392,4 +399,4 @@ "uid": "api-detailed-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/blend-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/blend-dashboard.json index 3ddda12..17177b9 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/blend-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/blend-dashboard.json @@ -99,7 +99,7 @@ "refId": "B" } ], - "title": "Message Flow Rate", + "title": "Message Flow Rate \u2014 events/sec", "type": "timeseries" }, { @@ -175,7 +175,7 @@ "refId": "A" } ], - "title": "Mix Packet Processing Rate", + "title": "Mix Packet Processing Rate \u2014 events/sec", "type": "timeseries" }, { @@ -213,7 +213,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -225,7 +227,7 @@ "refId": "A" } ], - "title": "Total Messages Sent", + "title": "Total Messages Sent \u2014 current", "type": "stat" }, { @@ -263,7 +265,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -275,7 +279,7 @@ "refId": "A" } ], - "title": "Total Messages Received", + "title": "Total Messages Received \u2014 current", "type": "stat" }, { @@ -313,7 +317,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -325,7 +331,7 @@ "refId": "A" } ], - "title": "Total Mix Packets Processed", + "title": "Total Mix Packets Processed \u2014 current", "type": "stat" }, { @@ -363,7 +369,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -375,7 +383,7 @@ "refId": "A" } ], - "title": "Connected Peers", + "title": "Connected Peers \u2014 peer count", "type": "stat" }, { @@ -451,7 +459,7 @@ "refId": "A" } ], - "title": "Peer Connectivity Over Time", + "title": "Peer Connectivity Over Time \u2014 current", "type": "timeseries" }, { @@ -527,7 +535,7 @@ "refId": "A" } ], - "title": "Failed Message Rate", + "title": "Failed Message Rate \u2014 events/sec", "type": "timeseries" }, { @@ -608,7 +616,7 @@ "refId": "B" } ], - "title": "Connection & Error Events", + "title": "Connection & Error Events \u2014 events/sec", "type": "timeseries" }, { @@ -646,7 +654,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -658,13 +668,18 @@ "refId": "A" } ], - "title": "Pending Outbound Messages", + "title": "Pending Outbound Messages \u2014 queue depth", "type": "stat" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "blend", "privacy", "mixnet"], + "tags": [ + "nomos", + "blend", + "privacy", + "mixnet" + ], "templating": { "list": [] }, @@ -678,4 +693,4 @@ "uid": "blend-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/consensus-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/consensus-dashboard.json index 9b3109f..45cf29c 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/consensus-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/consensus-dashboard.json @@ -56,7 +56,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -68,7 +70,7 @@ "refId": "A" } ], - "title": "Tip Height", + "title": "Tip Height \u2014 height", "type": "stat" }, { @@ -106,7 +108,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -118,7 +122,7 @@ "refId": "A" } ], - "title": "Finalized Height (LIB)", + "title": "Finalized Height (LIB) \u2014 height", "type": "stat" }, { @@ -156,7 +160,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -168,7 +174,7 @@ "refId": "A" } ], - "title": "Current Epoch", + "title": "Current Epoch \u2014 epoch", "type": "stat" }, { @@ -206,7 +212,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -218,7 +226,7 @@ "refId": "A" } ], - "title": "Current Slot", + "title": "Current Slot \u2014 slot", "type": "stat" }, { @@ -299,7 +307,7 @@ "refId": "B" } ], - "title": "Block Heights Over Time", + "title": "Block Heights Over Time \u2014 height", "type": "timeseries" }, { @@ -375,7 +383,7 @@ "refId": "A" } ], - "title": "Block Import Rate", + "title": "Block Import Rate \u2014 events/sec", "type": "timeseries" }, { @@ -456,7 +464,7 @@ "refId": "B" } ], - "title": "Network Connectivity", + "title": "Network Connectivity \u2014 conn count", "type": "timeseries" }, { @@ -494,7 +502,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -506,7 +516,7 @@ "refId": "A" } ], - "title": "Active Branches", + "title": "Active Branches \u2014 current", "type": "stat" }, { @@ -544,7 +554,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -556,7 +568,7 @@ "refId": "A" } ], - "title": "Txs in Last Block", + "title": "Txs in Last Block \u2014 current", "type": "stat" }, { @@ -594,7 +606,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -606,7 +620,7 @@ "refId": "A" } ], - "title": "Total Txs Processed", + "title": "Total Txs Processed \u2014 current", "type": "stat" }, { @@ -644,7 +658,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -661,7 +677,7 @@ "refId": "B" } ], - "title": "Network Tx Totals", + "title": "Network Tx Totals \u2014 current", "type": "stat" }, { @@ -737,7 +753,7 @@ "refId": "A" } ], - "title": "Transactions per Block", + "title": "Transactions per Block \u2014 current", "type": "timeseries" }, { @@ -813,7 +829,7 @@ "refId": "A" } ], - "title": "Transaction Throughput (TPS)", + "title": "Transaction Throughput (TPS) \u2014 events/sec", "type": "timeseries" }, { @@ -894,7 +910,7 @@ "refId": "B" } ], - "title": "Consensus Timing Health", + "title": "Consensus Timing Health \u2014 events/sec", "type": "timeseries" }, { @@ -932,7 +948,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -949,7 +967,7 @@ "refId": "B" } ], - "title": "Current Epoch & Slot", + "title": "Current Epoch & Slot \u2014 slot", "type": "stat" }, { @@ -1035,7 +1053,7 @@ "refId": "C" } ], - "title": "Network Message Processing", + "title": "Network Message Processing \u2014 events/sec", "type": "timeseries" }, { @@ -1073,7 +1091,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -1090,7 +1110,7 @@ "refId": "B" } ], - "title": "Network Subscription Errors", + "title": "Network Subscription Errors \u2014 events/sec", "type": "stat" }, { @@ -1176,7 +1196,7 @@ "refId": "C" } ], - "title": "Orphan Block Processing", + "title": "Orphan Block Processing \u2014 events/sec", "type": "timeseries" }, { @@ -1214,7 +1234,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -1226,111 +1248,7 @@ "refId": "A" } ], - "title": "Orphan Block Queue Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [ - { - "options": { - "from": 0, - "result": { - "color": "green", - "index": 0, - "text": "GOOD" - }, - "to": 5 - }, - "type": "range" - }, - { - "options": { - "from": 5, - "result": { - "color": "yellow", - "index": 1, - "text": "SLOW" - }, - "to": 20 - }, - "type": "range" - }, - { - "options": { - "from": 20, - "result": { - "color": "red", - "index": 2, - "text": "STALLED" - }, - "to": null - }, - "type": "range" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - }, - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 64 - }, - "id": 20, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_tip_height - consensus_finalized_height", - "legendFormat": "Unfinalized Block Gap - {{job}}", - "refId": "A" - } - ], - "title": "🚦 Consensus Health (Finalization Gap)", + "title": "Orphan Block Queue Status \u2014 queue depth", "type": "stat" }, { @@ -1406,13 +1324,17 @@ "refId": "A" } ], - "title": "⏱️ Block Production Timing", + "title": "\u23f1\ufe0f Block Production Timing \u2014 time since last", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "consensus", "cryptarchia"], + "tags": [ + "nomos", + "consensus", + "cryptarchia" + ], "templating": { "list": [] }, @@ -1426,4 +1348,4 @@ "uid": "consensus-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/da-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/da-dashboard.json index 542aa19..51d6151 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/da-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/da-dashboard.json @@ -99,7 +99,7 @@ "refId": "B" } ], - "title": "Sample Verification Rate", + "title": "Sample Verification Rate \u2014 events/sec", "type": "timeseries" }, { @@ -180,7 +180,7 @@ "refId": "B" } ], - "title": "Blob Request/Response Rate", + "title": "Blob Request/Response Rate \u2014 events/sec", "type": "timeseries" }, { @@ -218,7 +218,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -230,7 +232,7 @@ "refId": "A" } ], - "title": "Total Samples Verified", + "title": "Total Samples Verified \u2014 current", "type": "stat" }, { @@ -268,7 +270,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -280,7 +284,7 @@ "refId": "A" } ], - "title": "Total Blob Requests", + "title": "Total Blob Requests \u2014 current", "type": "stat" }, { @@ -318,7 +322,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -330,7 +336,7 @@ "refId": "A" } ], - "title": "Total Blob Responses", + "title": "Total Blob Responses \u2014 current", "type": "stat" }, { @@ -416,7 +422,7 @@ "refId": "C" } ], - "title": "DA Network Connections", + "title": "DA Network Connections \u2014 conn count", "type": "timeseries" }, { @@ -492,13 +498,18 @@ "refId": "A" } ], - "title": "Sampling Success Rate", + "title": "Sampling Success Rate \u2014 events/sec", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "data-availability", "da", "sampling"], + "tags": [ + "nomos", + "data-availability", + "da", + "sampling" + ], "templating": { "list": [] }, @@ -512,4 +523,4 @@ "uid": "da-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/issues-diagnostics-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/issues-diagnostics-dashboard.json index 405d519..c365d63 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/issues-diagnostics-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/issues-diagnostics-dashboard.json @@ -73,7 +73,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -110,7 +112,7 @@ "refId": "F" } ], - "title": "🚨 Critical Failure Rates (per minute)", + "title": "\ud83d\udea8 Critical Failure Rates (per minute) \u2014 events/sec", "type": "stat" }, { @@ -196,7 +198,7 @@ "refId": "C" } ], - "title": "📊 Failure Rates (Percentage)", + "title": "\ud83d\udcca Failure Rates (Percentage) \u2014 events/sec", "type": "timeseries" }, { @@ -282,7 +284,7 @@ "refId": "C" } ], - "title": "🔐 KMS Failures", + "title": "\ud83d\udd10 KMS Failures \u2014 events/sec", "type": "timeseries" }, { @@ -373,7 +375,7 @@ "refId": "D" } ], - "title": "📋 SDP (Service Provider) Failures", + "title": "\ud83d\udccb SDP (Service Provider) Failures \u2014 events/sec", "type": "timeseries" }, { @@ -464,7 +466,7 @@ "refId": "D" } ], - "title": "⏰ Timing/Synchronization Issues", + "title": "\u23f0 Timing/Synchronization Issues \u2014 events/sec", "type": "timeseries" }, { @@ -550,7 +552,7 @@ "refId": "C" } ], - "title": "🌐 Network/Subscription Errors", + "title": "\ud83c\udf10 Network/Subscription Errors \u2014 events/sec", "type": "timeseries" }, { @@ -605,7 +607,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -627,7 +631,7 @@ "refId": "C" } ], - "title": "⚠️ Potential Bottlenecks", + "title": "\u26a0\ufe0f Potential Bottlenecks \u2014 finalization gap", "type": "stat" }, { @@ -703,13 +707,120 @@ "refId": "A" } ], - "title": "🗄️ Top Storage Operation Failures", + "title": "\ud83d\uddc4\ufe0f Top Storage Operation Failures \u2014 events/sec", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(consensus_proposal_reconstruct_seconds_bucket[5m]))", + "legendFormat": "Reconstruct p95 - {{job}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(consensus_apply_block_seconds_bucket[5m]))", + "legendFormat": "Apply block p95 - {{job}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, rate(consensus_block_blob_validation_seconds_bucket[5m]))", + "legendFormat": "Blob validation p95 {{mode}} - {{job}}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.95, rate(chainsync_request_tip_seconds_bucket[5m]))", + "legendFormat": "ChainSync tip p95 - {{job}}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.95, rate(chainsync_download_blocks_seconds_bucket[5m]))", + "legendFormat": "ChainSync blocks p95 - {{job}}", + "refId": "E" + }, + { + "expr": "histogram_quantile(0.95, rate(orphan_blocks_parent_fetch_seconds_bucket[5m]))", + "legendFormat": "Orphan parent fetch p95 - {{job}}", + "refId": "F" + } + ], + "title": "\u23f1\ufe0f Consensus Pipeline Latency (p95) \u2014 p95 latency", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "diagnostics", "errors", "issues", "alerting"], + "tags": [ + "nomos", + "diagnostics", + "errors", + "issues", + "alerting" + ], "templating": { "list": [] }, @@ -719,8 +830,8 @@ }, "timepicker": {}, "timezone": "", - "title": "🚨 Issues & Diagnostics Dashboard", + "title": "\ud83d\udea8 Issues & Diagnostics Dashboard", "uid": "issues-diagnostics-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/mempool-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/mempool-dashboard.json index 983f716..43afe5e 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/mempool-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/mempool-dashboard.json @@ -56,7 +56,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -68,7 +70,7 @@ "refId": "A" } ], - "title": "Pending Transactions", + "title": "Pending Transactions \u2014 queue depth", "type": "stat" }, { @@ -106,7 +108,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -118,7 +122,7 @@ "refId": "A" } ], - "title": "Total Added", + "title": "Total Added \u2014 current", "type": "stat" }, { @@ -156,7 +160,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -168,7 +174,7 @@ "refId": "A" } ], - "title": "Total Removed", + "title": "Total Removed \u2014 current", "type": "stat" }, { @@ -244,7 +250,7 @@ "refId": "A" } ], - "title": "Pending Transactions Over Time", + "title": "Pending Transactions Over Time \u2014 queue depth", "type": "timeseries" }, { @@ -325,7 +331,7 @@ "refId": "B" } ], - "title": "Transaction Processing Rate", + "title": "Transaction Processing Rate \u2014 events/sec", "type": "timeseries" }, { @@ -406,7 +412,7 @@ "refId": "B" } ], - "title": "Cumulative Transaction Counts", + "title": "Cumulative Transaction Counts \u2014 current", "type": "timeseries" }, { @@ -482,7 +488,7 @@ "refId": "A" } ], - "title": "Mempool Processing Efficiency", + "title": "Mempool Processing Efficiency \u2014 events/sec", "type": "timeseries" }, { @@ -520,7 +526,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -532,13 +540,17 @@ "refId": "A" } ], - "title": "TX Service Mempool Metrics", + "title": "TX Service Mempool Metrics \u2014 current", "type": "stat" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "mempool", "transactions"], + "tags": [ + "nomos", + "mempool", + "transactions" + ], "templating": { "list": [] }, @@ -552,4 +564,4 @@ "uid": "mempool-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/overview-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/overview-dashboard.json index ae9ab93..c591d69 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/overview-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/overview-dashboard.json @@ -56,7 +56,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -68,7 +70,7 @@ "refId": "A" } ], - "title": "Chain Tip", + "title": "Chain Tip \u2014 height", "type": "stat" }, { @@ -106,7 +108,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -118,7 +122,7 @@ "refId": "A" } ], - "title": "Current Epoch", + "title": "Current Epoch \u2014 epoch", "type": "stat" }, { @@ -156,7 +160,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -168,7 +174,7 @@ "refId": "A" } ], - "title": "Consensus Peers", + "title": "Consensus Peers \u2014 peer count", "type": "stat" }, { @@ -206,7 +212,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -218,7 +226,7 @@ "refId": "A" } ], - "title": "Blend Peers", + "title": "Blend Peers \u2014 peer count", "type": "stat" }, { @@ -256,7 +264,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -268,7 +278,7 @@ "refId": "A" } ], - "title": "Pending Txs", + "title": "Pending Txs \u2014 queue depth", "type": "stat" }, { @@ -306,7 +316,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -318,7 +330,7 @@ "refId": "A" } ], - "title": "DA Samples", + "title": "DA Samples \u2014 current", "type": "stat" }, { @@ -399,7 +411,7 @@ "refId": "B" } ], - "title": "Consensus Heights", + "title": "Consensus Heights \u2014 height", "type": "timeseries" }, { @@ -485,7 +497,7 @@ "refId": "C" } ], - "title": "Processing Rates", + "title": "Processing Rates \u2014 events/sec", "type": "timeseries" }, { @@ -523,7 +535,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -545,7 +559,7 @@ "refId": "C" } ], - "title": "Network Transaction Totals", + "title": "Network Transaction Totals \u2014 current", "type": "stat" }, { @@ -626,7 +640,7 @@ "refId": "B" } ], - "title": "Blend Message Flow", + "title": "Blend Message Flow \u2014 events/sec", "type": "timeseries" }, { @@ -707,7 +721,7 @@ "refId": "B" } ], - "title": "Network Connectivity", + "title": "Network Connectivity \u2014 current", "type": "timeseries" }, { @@ -745,7 +759,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -757,7 +773,7 @@ "refId": "A" } ], - "title": "Current Epoch", + "title": "Current Epoch \u2014 epoch", "type": "stat" }, { @@ -795,7 +811,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -807,7 +825,7 @@ "refId": "A" } ], - "title": "Current Slot", + "title": "Current Slot \u2014 slot", "type": "stat" }, { @@ -893,7 +911,7 @@ "refId": "C" } ], - "title": "Service Activity Rates", + "title": "Service Activity Rates \u2014 events/sec", "type": "timeseries" }, { @@ -967,7 +985,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -979,13 +999,17 @@ "refId": "A" } ], - "title": "🚦 Overall System Health", + "title": "\ud83d\udea6 Overall System Health \u2014 events/sec", "type": "stat" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "overview", "summary"], + "tags": [ + "nomos", + "overview", + "summary" + ], "templating": { "list": [] }, @@ -999,4 +1023,4 @@ "uid": "overview-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/services-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/services-dashboard.json deleted file mode 100644 index 6eb576d..0000000 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/services-dashboard.json +++ /dev/null @@ -1,676 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "kms_register_success_total", - "legendFormat": "Registered Keys - {{job}}", - "refId": "A" - } - ], - "title": "KMS Keys Registered", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 0 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "kms_sign_success_total", - "legendFormat": "Signatures - {{job}}", - "refId": "A" - } - ], - "title": "KMS Signatures Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 0 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sdp_declarations_total", - "legendFormat": "Declarations - {{job}}", - "refId": "A" - } - ], - "title": "SDP Declarations Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "sdp_activity_posts_total", - "legendFormat": "Activities - {{job}}", - "refId": "A" - } - ], - "title": "SDP Activities Total", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "ops" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(kms_sign_requests_total[5m])", - "legendFormat": "Sign Requests/sec - {{strategy}}", - "refId": "A" - }, - { - "expr": "rate(kms_register_requests_total[5m])", - "legendFormat": "Register Requests/sec", - "refId": "B" - }, - { - "expr": "rate(kms_public_key_requests_total[5m])", - "legendFormat": "PubKey Requests/sec", - "refId": "C" - } - ], - "title": "KMS Request Rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "percent" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(kms_sign_failures_total[5m]) / rate(kms_sign_requests_total[5m]) * 100", - "legendFormat": "KMS Sign Error % - {{strategy}}", - "refId": "A" - }, - { - "expr": "rate(kms_register_failures_total[5m]) / rate(kms_register_requests_total[5m]) * 100", - "legendFormat": "KMS Register Error %", - "refId": "B" - } - ], - "title": "KMS Error Rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "ops" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(sdp_declarations_total[5m])", - "legendFormat": "Declarations/sec", - "refId": "A" - }, - { - "expr": "rate(sdp_activity_posts_total[5m])", - "legendFormat": "Activities/sec", - "refId": "B" - }, - { - "expr": "rate(sdp_withdrawals_total[5m])", - "legendFormat": "Withdrawals/sec", - "refId": "C" - } - ], - "title": "SDP Operation Rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "ops" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(sdp_declaration_success_total[5m])", - "legendFormat": "Declaration Success/sec", - "refId": "A" - }, - { - "expr": "rate(sdp_declaration_tx_failures_total[5m])", - "legendFormat": "Declaration TX Failures/sec", - "refId": "B" - }, - { - "expr": "rate(sdp_activity_success_total[5m])", - "legendFormat": "Activity Success/sec", - "refId": "C" - }, - { - "expr": "rate(sdp_withdrawal_success_total[5m])", - "legendFormat": "Withdrawal Success/sec", - "refId": "D" - } - ], - "title": "SDP Success/Failure Rates", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "ops" - } - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(time_subscription_requests_total[5m])", - "legendFormat": "Time Subscriptions/sec", - "refId": "A" - }, - { - "expr": "rate(network_pubsub_subscriptions_total[5m])", - "legendFormat": "PubSub Subscriptions/sec", - "refId": "B" - }, - { - "expr": "rate(network_chainsync_subscriptions_total[5m])", - "legendFormat": "ChainSync Subscriptions/sec", - "refId": "C" - }, - { - "expr": "rate(sdp_subscriptions_total[5m])", - "legendFormat": "SDP Subscriptions/sec", - "refId": "D" - } - ], - "title": "Service Subscription Activity", - "type": "timeseries" - } - ], - "refresh": "5s", - "schemaVersion": 40, - "tags": ["nomos", "services", "kms", "sdp", "time"], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Services Dashboard", - "uid": "services-dashboard", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/storage-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/storage-dashboard.json index e2bbd4e..1fc3080 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/storage-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/storage-dashboard.json @@ -94,7 +94,7 @@ "refId": "A" } ], - "title": "DA Storage Operations Rate", + "title": "DA Storage Operations Rate \u2014 events/sec", "type": "timeseries" }, { @@ -170,7 +170,7 @@ "refId": "A" } ], - "title": "Chain Storage Operations Rate", + "title": "Chain Storage Operations Rate \u2014 events/sec", "type": "timeseries" }, { @@ -246,7 +246,7 @@ "refId": "A" } ], - "title": "DA Storage Failures Rate", + "title": "DA Storage Failures Rate \u2014 events/sec", "type": "timeseries" }, { @@ -322,7 +322,7 @@ "refId": "A" } ], - "title": "Chain Storage Failures Rate", + "title": "Chain Storage Failures Rate \u2014 events/sec", "type": "timeseries" }, { @@ -360,7 +360,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -372,7 +374,7 @@ "refId": "A" } ], - "title": "DA Storage Operations Total", + "title": "DA Storage Operations Total \u2014 current", "type": "stat" }, { @@ -410,7 +412,9 @@ "orientation": "auto", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "" }, "textMode": "auto" @@ -422,13 +426,17 @@ "refId": "A" } ], - "title": "Chain Storage Operations Total", + "title": "Chain Storage Operations Total \u2014 current", "type": "stat" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "storage", "rocksdb"], + "tags": [ + "nomos", + "storage", + "rocksdb" + ], "templating": { "list": [] }, @@ -442,4 +450,4 @@ "uid": "storage-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/storage-detailed-dashboard.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/storage-detailed-dashboard.json index 2ad6eb1..ac075b8 100644 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/storage-detailed-dashboard.json +++ b/testing-framework/assets/stack/monitoring/grafana/dashboards/storage-detailed-dashboard.json @@ -99,7 +99,7 @@ "refId": "B" } ], - "title": "DA: Get Light Share", + "title": "DA: Get Light Share \u2014 events/sec", "type": "timeseries" }, { @@ -180,7 +180,7 @@ "refId": "B" } ], - "title": "DA: Store Light Share", + "title": "DA: Store Light Share \u2014 events/sec", "type": "timeseries" }, { @@ -261,7 +261,7 @@ "refId": "B" } ], - "title": "DA: Get Shared Commitments", + "title": "DA: Get Shared Commitments \u2014 events/sec", "type": "timeseries" }, { @@ -342,7 +342,7 @@ "refId": "B" } ], - "title": "DA: Store Shared Commitments", + "title": "DA: Store Shared Commitments \u2014 events/sec", "type": "timeseries" }, { @@ -423,7 +423,7 @@ "refId": "B" } ], - "title": "DA: Store Transaction", + "title": "DA: Store Transaction \u2014 events/sec", "type": "timeseries" }, { @@ -504,7 +504,7 @@ "refId": "B" } ], - "title": "DA: Get Transaction", + "title": "DA: Get Transaction \u2014 events/sec", "type": "timeseries" }, { @@ -585,7 +585,7 @@ "refId": "B" } ], - "title": "Chain: Get Block", + "title": "Chain: Get Block \u2014 events/sec", "type": "timeseries" }, { @@ -666,7 +666,7 @@ "refId": "B" } ], - "title": "Chain: Store Block", + "title": "Chain: Store Block \u2014 events/sec", "type": "timeseries" }, { @@ -747,7 +747,7 @@ "refId": "B" } ], - "title": "Chain: Remove Block", + "title": "Chain: Remove Block \u2014 events/sec", "type": "timeseries" }, { @@ -828,7 +828,7 @@ "refId": "B" } ], - "title": "Chain: Store Transactions", + "title": "Chain: Store Transactions \u2014 events/sec", "type": "timeseries" }, { @@ -904,7 +904,7 @@ "refId": "A" } ], - "title": "Chain: Get Transactions", + "title": "Chain: Get Transactions \u2014 events/sec", "type": "timeseries" }, { @@ -985,13 +985,18 @@ "refId": "B" } ], - "title": "Chain: Remove Transactions", + "title": "Chain: Remove Transactions \u2014 events/sec", "type": "timeseries" } ], "refresh": "5s", "schemaVersion": 40, - "tags": ["nomos", "storage", "rocksdb", "detailed"], + "tags": [ + "nomos", + "storage", + "rocksdb", + "detailed" + ], "templating": { "list": [] }, @@ -1005,4 +1010,4 @@ "uid": "storage-detailed-dashboard", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/testing-framework/assets/stack/monitoring/grafana/dashboards/testnet_metrics.json b/testing-framework/assets/stack/monitoring/grafana/dashboards/testnet_metrics.json deleted file mode 100644 index 11dcd51..0000000 --- a/testing-framework/assets/stack/monitoring/grafana/dashboards/testnet_metrics.json +++ /dev/null @@ -1,1206 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "mempool_transactions_pending", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Mempool: Pending DA blobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "rate(consensus_blocks_imported_total[5m])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Consensus: Block Import Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 0, - "y": 16 - }, - "id": 3, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_tip_height", - "refId": "A" - } - ], - "title": "Consensus Tip Height", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 4, - "y": 16 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_finalized_height", - "refId": "A" - } - ], - "title": "Finalized Height", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 8, - "y": 16 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_current_epoch", - "refId": "A" - } - ], - "title": "Current Epoch", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "reqps" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(da_samples_verified_total[5m])", - "legendFormat": "Verified/sec", - "refId": "A" - }, - { - "expr": "rate(da_samples_failed_total[5m])", - "legendFormat": "Failed/sec", - "refId": "B" - } - ], - "title": "DA Sample Verification Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 7, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "consensus_peers_connected", - "legendFormat": "Connected Peers", - "refId": "A" - }, - { - "expr": "consensus_connections_total", - "legendFormat": "Total Connections", - "refId": "B" - } - ], - "title": "Consensus Network", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "blend_peers_connected", - "legendFormat": "Connected Peers", - "refId": "A" - } - ], - "title": "Blend Network", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 0, - "y": 32 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_tip_height", - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Consensus Tip Height", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 4, - "y": 32 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_finalized_height", - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Consensus Finalized Height", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 8, - "y": 32 - }, - "id": 11, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_current_epoch", - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Current Epoch", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 12, - "y": 32 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_current_slot", - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Current Slot", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "vis": false - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 16, - "y": 32 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "values": false, - "calcs": ["lastNotNull"], - "fields": "" - }, - "textMode": "auto" - }, - "targets": [ - { - "expr": "consensus_branches_count", - "legendFormat": "{{job}}", - "refId": "A" - } - ], - "title": "Consensus Branches", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "consensus_peers_connected", - "legendFormat": "Connected Peers - {{job}}", - "refId": "A" - }, - { - "expr": "consensus_connections_total", - "legendFormat": "Total Connections - {{job}}", - "refId": "B" - } - ], - "title": "Consensus Network Connections", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "short" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "mempool_transactions_pending", - "legendFormat": "Pending - {{job}}", - "refId": "A" - }, - { - "expr": "rate(mempool_transactions_added_total[5m])", - "legendFormat": "Added/sec - {{job}}", - "refId": "B" - }, - { - "expr": "rate(mempool_transactions_removed_total[5m])", - "legendFormat": "Removed/sec - {{job}}", - "refId": "C" - } - ], - "title": "Mempool Transactions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "reqps" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(da_blob_requests_total[5m])", - "legendFormat": "Requests/sec - {{job}}", - "refId": "A" - }, - { - "expr": "rate(da_blob_responses_total[5m])", - "legendFormat": "Responses/sec - {{job}}", - "refId": "B" - } - ], - "title": "DA Blob Request/Response Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "unit": "msgps" - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "expr": "rate(blend_messages_sent_total[5m])", - "legendFormat": "Sent/sec - {{job}}", - "refId": "A" - }, - { - "expr": "rate(blend_messages_received_total[5m])", - "legendFormat": "Received/sec - {{job}}", - "refId": "B" - }, - { - "expr": "rate(blend_mix_packets_processed_total[5m])", - "legendFormat": "Mix Packets/sec - {{job}}", - "refId": "C" - } - ], - "title": "Blend Message Flow Rate", - "type": "timeseries" - } - ], - "preload": false, - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Testnet Metrics", - "uid": "ce6ebepwk737kf", - "version": 5, - "weekStart": "" -} diff --git a/testing-framework/assets/stack/scripts/build_test_image.sh b/testing-framework/assets/stack/scripts/build_test_image.sh index 0010f59..c94c135 100755 --- a/testing-framework/assets/stack/scripts/build_test_image.sh +++ b/testing-framework/assets/stack/scripts/build_test_image.sh @@ -1,23 +1,30 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail # Builds the testnet image with circuits. Prefers a local circuits bundle # (tests/kzgrs/kzgrs_test_params) or a custom override; otherwise downloads # from logos-co/nomos-circuits. +# Always run under bash; bail out if someone invokes via sh. +if [ -z "${BASH_VERSION:-}" ]; then + exec bash "$0" "$@" +fi + ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" -if [ -f "${ROOT_DIR}/versions.env" ]; then - # shellcheck disable=SC1091 - . "${ROOT_DIR}/versions.env" -fi -if [ -f "${ROOT_DIR}/paths.env" ]; then - # shellcheck disable=SC1091 - . "${ROOT_DIR}/paths.env" -fi +# shellcheck disable=SC1091 +. "${ROOT_DIR}/scripts/lib/common.sh" + +common::require_file "${ROOT_DIR}/versions.env" +# shellcheck disable=SC1091 +. "${ROOT_DIR}/versions.env" +common::maybe_source "${ROOT_DIR}/paths.env" + DOCKERFILE_PATH="${ROOT_DIR}/testing-framework/assets/stack/Dockerfile" IMAGE_TAG="${IMAGE_TAG:-logos-blockchain-testing:local}" -VERSION="${VERSION:-v0.3.1}" +DEFAULT_VERSION="${VERSION:-v0.3.1}" +VERSION="${VERSION:-${DEFAULT_VERSION}}" KZG_DIR_REL="${NOMOS_KZG_DIR_REL:-testing-framework/assets/stack/kzgrs_test_params}" +CIRCUITS_DIR_HOST="${ROOT_DIR}/${KZG_DIR_REL}" CIRCUITS_OVERRIDE="${CIRCUITS_OVERRIDE:-${KZG_DIR_REL}}" CIRCUITS_PLATFORM="${CIRCUITS_PLATFORM:-${COMPOSE_CIRCUITS_PLATFORM:-}}" if [ -z "${CIRCUITS_PLATFORM}" ]; then @@ -27,14 +34,14 @@ if [ -z "${CIRCUITS_PLATFORM}" ]; then *) CIRCUITS_PLATFORM="linux-x86_64" ;; esac fi -NOMOS_NODE_REV="${NOMOS_NODE_REV:-d2dd5a5084e1daef4032562c77d41de5e4d495f8}" +NOMOS_NODE_REV="${NOMOS_NODE_REV:?Missing NOMOS_NODE_REV in versions.env or env}" echo "Workspace root: ${ROOT_DIR}" echo "Image tag: ${IMAGE_TAG}" echo "Circuits override: ${CIRCUITS_OVERRIDE:-}" echo "Circuits version (fallback download): ${VERSION}" echo "Circuits platform: ${CIRCUITS_PLATFORM}" -echo "Bundle tar (if used): ${NOMOS_BINARIES_TAR:-.tmp/nomos-binaries-linux-${VERSION}.tar.gz}" +echo "Bundle tar (if used): ${NOMOS_BINARIES_TAR:- ${ROOT_DIR}/.tmp/nomos-binaries-linux-${VERSION}.tar.gz}" # If prebuilt binaries are missing, restore them from a bundle tarball instead of # rebuilding nomos inside the image. @@ -45,23 +52,25 @@ TAR_PATH="${NOMOS_BINARIES_TAR:-${DEFAULT_LINUX_TAR}}" if [ ! -x "${BIN_DST}/nomos-node" ] || [ ! -x "${BIN_DST}/nomos-executor" ]; then if [ -f "${TAR_PATH}" ]; then echo "Restoring binaries/circuits from ${TAR_PATH}" - tmp_extract="$(mktemp -d)" + tmp_extract="$(common::tmpdir nomos-bundle-extract.XXXXXX)" tar -xzf "${TAR_PATH}" -C "${tmp_extract}" if [ -f "${tmp_extract}/artifacts/nomos-node" ] && [ -f "${tmp_extract}/artifacts/nomos-executor" ]; then mkdir -p "${BIN_DST}" cp "${tmp_extract}/artifacts/nomos-node" "${tmp_extract}/artifacts/nomos-executor" "${tmp_extract}/artifacts/nomos-cli" "${BIN_DST}/" else - echo "ERROR: Bundle ${TAR_PATH} missing binaries under artifacts/" >&2 - exit 1 + common::die "Bundle ${TAR_PATH} missing binaries under artifacts/" fi if [ -d "${tmp_extract}/artifacts/circuits" ]; then - mkdir -p "${KZG_DIR_REL}" - rsync -a --delete "${tmp_extract}/artifacts/circuits/" "${KZG_DIR_REL}/" + mkdir -p "${CIRCUITS_DIR_HOST}" + if command -v rsync >/dev/null 2>&1; then + rsync -a --delete "${tmp_extract}/artifacts/circuits/" "${CIRCUITS_DIR_HOST}/" + else + cp -a "${tmp_extract}/artifacts/circuits/." "${CIRCUITS_DIR_HOST}/" + fi fi rm -rf "${tmp_extract}" else - echo "ERROR: Prebuilt binaries missing and bundle tar not found at ${TAR_PATH}" >&2 - exit 1 + common::die "Prebuilt binaries missing and bundle tar not found at ${TAR_PATH}" fi fi diff --git a/testing-framework/core/src/scenario/http_probe.rs b/testing-framework/core/src/scenario/http_probe.rs index 13cc116..be67bf3 100644 --- a/testing-framework/core/src/scenario/http_probe.rs +++ b/testing-framework/core/src/scenario/http_probe.rs @@ -4,7 +4,7 @@ use futures::future::try_join_all; use nomos_http_api_common::paths; use reqwest::Client as ReqwestClient; use thiserror::Error; -use tokio::time::{sleep, timeout}; +use tokio::time::{Instant, sleep}; use tracing::{debug, info}; /// Role used for labelling readiness probes. @@ -115,24 +115,61 @@ async fn wait_for_single_port( ) -> Result<(), HttpReadinessError> { let url = format!("http://{host}:{port}{}", paths::CRYPTARCHIA_INFO); debug!(role = role.label(), %url, "probing HTTP endpoint"); - let probe = async { - loop { - let is_ready = client - .get(&url) - .send() - .await - .map(|response| response.status().is_success()) - .unwrap_or(false); + let start = Instant::now(); + let deadline = start + timeout_duration; + let mut attempts: u64 = 0; - if is_ready { - return; + loop { + attempts += 1; + + let last_failure: Option = match client.get(&url).send().await { + Ok(response) if response.status().is_success() => { + info!( + role = role.label(), + port, + host, + %url, + attempts, + elapsed_ms = start.elapsed().as_millis(), + "HTTP readiness confirmed" + ); + return Ok(()); } + Ok(response) => { + let status = response.status(); + Some(format!("HTTP {status}")) + } + Err(error) => Some(format!("request error: {error}")), + }; - sleep(poll_interval).await; + if attempts == 1 || attempts % 10 == 0 { + debug!( + role = role.label(), + port, + host, + %url, + attempts, + elapsed_ms = start.elapsed().as_millis(), + last_failure = last_failure.as_deref().unwrap_or(""), + "HTTP readiness not yet available" + ); } - }; - timeout(timeout_duration, probe) - .await - .map_err(|_| HttpReadinessError::new(role, port, timeout_duration)) + if Instant::now() >= deadline { + info!( + role = role.label(), + port, + host, + %url, + attempts, + elapsed_ms = start.elapsed().as_millis(), + timeout_secs = timeout_duration.as_secs_f32(), + last_failure = last_failure.as_deref().unwrap_or(""), + "HTTP readiness timed out" + ); + return Err(HttpReadinessError::new(role, port, timeout_duration)); + } + + sleep(poll_interval).await; + } } diff --git a/testing-framework/runners/k8s/helm/nomos-runner/grafana/dashboards/.gitignore b/testing-framework/runners/k8s/helm/nomos-runner/grafana/dashboards/.gitignore new file mode 100644 index 0000000..0827618 --- /dev/null +++ b/testing-framework/runners/k8s/helm/nomos-runner/grafana/dashboards/.gitignore @@ -0,0 +1,2 @@ +*.json +!.gitignore diff --git a/testing-framework/runners/k8s/helm/nomos-runner/grafana/dashboards/testnet_metrics.json b/testing-framework/runners/k8s/helm/nomos-runner/grafana/dashboards/testnet_metrics.json deleted file mode 100644 index 454c260..0000000 --- a/testing-framework/runners/k8s/helm/nomos-runner/grafana/dashboards/testnet_metrics.json +++ /dev/null @@ -1,237 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "disableTextWrap": false, - "editorMode": "builder", - "expr": "da_mempool_pending_items", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Mempool: Pending DA blobs", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "consensus_processed_blocks", - "fullMetaSearch": false, - "includeNullMetadata": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Consensus: Processed Blocks", - "type": "timeseries" - } - ], - "preload": false, - "schemaVersion": 40, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "title": "Testnet Metrics", - "uid": "ce6ebepwk737kf", - "version": 5, - "weekStart": "" -} diff --git a/testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-configmap.yaml b/testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-configmap.yaml index 97e729d..e7c3cfa 100644 --- a/testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-configmap.yaml +++ b/testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-configmap.yaml @@ -9,9 +9,12 @@ data: datasources: - name: Prometheus type: prometheus - url: http://{{ include "nomos-runner.fullname" . }}-prometheus:9090 access: proxy isDefault: true + uid: PBFA97CFB590B2093 + orgId: 1 + url: http://prometheus:9090 + editable: true dashboards.yaml: | apiVersion: 1 providers: @@ -23,6 +26,8 @@ data: editable: true options: path: /var/lib/grafana/dashboards - testnet_metrics.json: | -{{ (.Files.Get "grafana/dashboards/testnet_metrics.json") | indent 4 }} +{{ range $path, $_ := .Files.Glob "grafana/dashboards/*.json" }} + {{ base $path }}: | +{{ $.Files.Get $path | indent 4 }} +{{ end }} {{- end }} diff --git a/testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-deployment.yaml b/testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-deployment.yaml index 3ef9741..8f0448d 100644 --- a/testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-deployment.yaml +++ b/testing-framework/runners/k8s/helm/nomos-runner/templates/grafana-deployment.yaml @@ -57,6 +57,8 @@ spec: configMap: name: {{ include "nomos-runner.fullname" . }}-grafana-config items: - - key: testnet_metrics.json - path: testnet_metrics.json +{{ range $path, $_ := .Files.Glob "grafana/dashboards/*.json" }} + - key: {{ base $path }} + path: {{ base $path }} +{{ end }} {{- end }} diff --git a/testing-framework/runners/k8s/src/deployer/orchestrator.rs b/testing-framework/runners/k8s/src/deployer/orchestrator.rs index 888cb44..25385c6 100644 --- a/testing-framework/runners/k8s/src/deployer/orchestrator.rs +++ b/testing-framework/runners/k8s/src/deployer/orchestrator.rs @@ -1,7 +1,6 @@ use anyhow::Error; use async_trait::async_trait; -use k8s_openapi::api::core::v1::Service; -use kube::{Client, api::Api}; +use kube::Client; use testing_framework_core::{ scenario::{BlockFeedTask, CleanupGuard, Deployer, MetricsError, RunContext, Runner, Scenario}, topology::generation::GeneratedTopology, @@ -14,12 +13,13 @@ use crate::{ cluster::{ ClusterEnvironment, NodeClientError, PortSpecs, RemoteReadinessError, build_node_clients, cluster_identifiers, collect_port_specs, ensure_cluster_readiness, - install_stack, kill_port_forwards, metrics_handle_from_port, wait_for_ports_or_cleanup, + install_stack, kill_port_forwards, metrics_handle_from_endpoint, + wait_for_ports_or_cleanup, }, helm::HelmError, }, lifecycle::{block_feed::spawn_block_feed_with, cleanup::RunnerCleanup}, - wait::{ClusterWaitError, PortForwardHandle}, + wait::{ClusterWaitError, HostPort, PortForwardHandle}, }; /// Deploys a scenario into Kubernetes using Helm charts and port-forwards. @@ -123,7 +123,7 @@ impl Deployer for K8sDeployer { } }; - let telemetry = match metrics_handle_from_port(cluster_prometheus_port(&cluster)) { + let telemetry = match metrics_handle_from_endpoint(cluster_prometheus_endpoint(&cluster)) { Ok(handle) => handle, Err(err) => { fail_cluster( @@ -145,50 +145,26 @@ impl Deployer for K8sDeployer { } }; - let node_host = crate::host::node_host(); - let prometheus_port = cluster_prometheus_port(&cluster); + let prometheus = cluster_prometheus_endpoint(&cluster); info!( - prometheus_url = %format!("http://{}:{}/", node_host, prometheus_port), + prometheus_url = %format!("http://{}:{}/", prometheus.host, prometheus.port), "prometheus endpoint available on host" ); - if let Some(grafana_port) = cluster_grafana_node_port( - &client, - cluster - .as_ref() - .expect("cluster must be available") - .namespace(), - cluster - .as_ref() - .expect("cluster must be available") - .release(), - ) - .await - { + if let Some(grafana) = cluster_grafana_endpoint(&cluster) { info!( - grafana_url = %format!("http://{}:{}/", node_host, grafana_port), - "grafana dashboard available via NodePort" + grafana_url = %format!("http://{}:{}/", grafana.host, grafana.port), + "grafana dashboard available on host" ); } if std::env::var("TESTNET_PRINT_ENDPOINTS").is_ok() { - let grafana_port = cluster_grafana_node_port( - &client, - cluster - .as_ref() - .expect("cluster must be available") - .namespace(), - cluster - .as_ref() - .expect("cluster must be available") - .release(), - ) - .await; + let grafana = cluster_grafana_endpoint(&cluster); println!( "TESTNET_ENDPOINTS prometheus=http://{}:{}/ grafana={}", - node_host, - prometheus_port, - grafana_port - .map(|port| format!("http://{}:{}/", node_host, port)) + prometheus.host, + prometheus.port, + grafana + .map(|endpoint| format!("http://{}:{}/", endpoint.host, endpoint.port)) .unwrap_or_else(|| "".to_string()) ); @@ -240,26 +216,18 @@ impl Deployer for K8sDeployer { } } -fn cluster_prometheus_port(cluster: &Option) -> u16 { +fn cluster_prometheus_endpoint(cluster: &Option) -> &HostPort { cluster .as_ref() .expect("cluster must be available") - .prometheus_port() + .prometheus_endpoint() } -async fn cluster_grafana_node_port(client: &Client, namespace: &str, release: &str) -> Option { - let services: Api = Api::namespaced(client.clone(), namespace); - let service_name = format!("{release}-grafana"); - let service = services.get(&service_name).await.ok()?; - let spec = service.spec?; - if spec.type_.as_deref() != Some("NodePort") { - return None; - } - let ports = spec.ports?; - ports.into_iter().find_map(|port| { - let node_port = port.node_port?; - u16::try_from(node_port).ok() - }) +fn cluster_grafana_endpoint(cluster: &Option) -> Option<&HostPort> { + cluster + .as_ref() + .expect("cluster must be available") + .grafana_endpoint() } async fn fail_cluster(cluster: &mut Option, reason: &str) { @@ -307,7 +275,7 @@ async fn setup_cluster( wait_for_ports_or_cleanup(client, &namespace, &release, specs, &mut cleanup_guard).await?; info!( - prometheus_port = cluster_ready.ports.prometheus, + prometheus = ?cluster_ready.ports.prometheus, "discovered prometheus endpoint" ); diff --git a/testing-framework/runners/k8s/src/infrastructure/assets.rs b/testing-framework/runners/k8s/src/infrastructure/assets.rs index ec615e0..cf8305d 100644 --- a/testing-framework/runners/k8s/src/infrastructure/assets.rs +++ b/testing-framework/runners/k8s/src/infrastructure/assets.rs @@ -53,6 +53,8 @@ pub enum AssetsError { MissingKzg { path: PathBuf }, #[error("missing Helm chart at {path}; ensure the repository is up-to-date")] MissingChart { path: PathBuf }, + #[error("missing Grafana dashboards source at {path}")] + MissingGrafanaDashboards { path: PathBuf }, #[error("failed to create temporary directory for rendered assets: {source}")] TempDir { #[source] @@ -109,6 +111,7 @@ pub fn prepare_assets(topology: &GeneratedTopology) -> Result None, }; let chart_path = helm_chart_path()?; + sync_grafana_dashboards(&root, &chart_path)?; let values_yaml = render_values_yaml(topology)?; let values_file = write_temp_file(tempdir.path(), "values.yaml", values_yaml)?; let image = env::var("NOMOS_TESTNET_IMAGE") @@ -147,6 +150,72 @@ const CFGSYNC_K8S_TIMEOUT_SECS: u64 = 300; const DEFAULT_GRAFANA_NODE_PORT: u16 = 30030; const DEFAULT_IN_IMAGE_KZG_PARAMS_PATH: &str = "/opt/nomos/kzg-params/kzgrs_test_params"; +fn sync_grafana_dashboards(root: &Path, chart_path: &Path) -> Result<(), AssetsError> { + let source_dir = stack_assets_root(root).join("monitoring/grafana/dashboards"); + let dest_dir = chart_path.join("grafana/dashboards"); + + if !source_dir.exists() { + return Err(AssetsError::MissingGrafanaDashboards { path: source_dir }); + } + + fs::create_dir_all(&dest_dir).map_err(|source| AssetsError::Io { + path: dest_dir.clone(), + source, + })?; + + let mut removed = 0usize; + for entry in fs::read_dir(&dest_dir).map_err(|source| AssetsError::Io { + path: dest_dir.clone(), + source, + })? { + let entry = entry.map_err(|source| AssetsError::Io { + path: dest_dir.clone(), + source, + })?; + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("json") { + continue; + } + fs::remove_file(&path).map_err(|source| AssetsError::Io { + path: path.clone(), + source, + })?; + removed += 1; + } + + let mut copied = 0usize; + for entry in fs::read_dir(&source_dir).map_err(|source| AssetsError::Io { + path: source_dir.clone(), + source, + })? { + let entry = entry.map_err(|source| AssetsError::Io { + path: source_dir.clone(), + source, + })?; + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("json") { + continue; + } + let file_name = path.file_name().unwrap_or_default(); + let dest_path = dest_dir.join(file_name); + fs::copy(&path, &dest_path).map_err(|source| AssetsError::Io { + path: dest_path.clone(), + source, + })?; + copied += 1; + } + + debug!( + source = %source_dir.display(), + dest = %dest_dir.display(), + removed, + copied, + "synced Grafana dashboards into Helm chart" + ); + + Ok(()) +} + fn render_cfgsync_config( root: &Path, topology: &GeneratedTopology, diff --git a/testing-framework/runners/k8s/src/infrastructure/cluster.rs b/testing-framework/runners/k8s/src/infrastructure/cluster.rs index 6546e46..f809e5d 100644 --- a/testing-framework/runners/k8s/src/infrastructure/cluster.rs +++ b/testing-framework/runners/k8s/src/infrastructure/cluster.rs @@ -12,11 +12,11 @@ use url::ParseError; use uuid::Uuid; use crate::{ - host::node_host, infrastructure::assets::RunnerAssets, lifecycle::{cleanup::RunnerCleanup, logs::dump_namespace_logs}, wait::{ - ClusterPorts, ClusterReady, NodeConfigPorts, PortForwardHandle, wait_for_cluster_ready, + ClusterPorts, ClusterReady, HostPort, NodeConfigPorts, PortForwardHandle, + wait_for_cluster_ready, }, }; @@ -32,11 +32,14 @@ pub struct ClusterEnvironment { namespace: String, release: String, cleanup: Option, + validator_host: String, + executor_host: String, validator_api_ports: Vec, validator_testing_ports: Vec, executor_api_ports: Vec, executor_testing_ports: Vec, - prometheus_port: u16, + prometheus: HostPort, + grafana: Option, port_forwards: Vec, } @@ -59,11 +62,14 @@ impl ClusterEnvironment { namespace, release, cleanup: Some(cleanup), + validator_host: ports.validator_host.clone(), + executor_host: ports.executor_host.clone(), validator_api_ports, validator_testing_ports, executor_api_ports, executor_testing_ports, - prometheus_port: ports.prometheus, + prometheus: ports.prometheus.clone(), + grafana: ports.grafana.clone(), port_forwards, } } @@ -89,16 +95,22 @@ impl ClusterEnvironment { ) } + #[allow(dead_code)] pub fn namespace(&self) -> &str { &self.namespace } + #[allow(dead_code)] pub fn release(&self) -> &str { &self.release } - pub fn prometheus_port(&self) -> u16 { - self.prometheus_port + pub fn prometheus_endpoint(&self) -> &HostPort { + &self.prometheus + } + + pub fn grafana_endpoint(&self) -> Option<&HostPort> { + self.grafana.as_ref() } pub fn validator_ports(&self) -> (&[u16], &[u16]) { @@ -185,7 +197,12 @@ pub fn build_node_clients(cluster: &ClusterEnvironment) -> Result, _>>()?; let executors = cluster @@ -194,7 +211,12 @@ pub fn build_node_clients(cluster: &ClusterEnvironment) -> Result, _>>()?; @@ -207,8 +229,8 @@ pub fn build_node_clients(cluster: &ClusterEnvironment) -> Result Result { - let url = cluster_host_url(port) +pub fn metrics_handle_from_endpoint(endpoint: &HostPort) -> Result { + let url = cluster_host_url(&endpoint.host, endpoint.port) .map_err(|err| MetricsError::new(format!("invalid prometheus url: {err}")))?; Metrics::from_prometheus(url) } @@ -221,10 +243,16 @@ pub async fn ensure_cluster_readiness( let (validator_api, validator_testing) = cluster.validator_ports(); let (executor_api, executor_testing) = cluster.executor_ports(); - let validator_urls = readiness_urls(validator_api, NodeRole::Validator)?; - let executor_urls = readiness_urls(executor_api, NodeRole::Executor)?; - let validator_membership_urls = readiness_urls(validator_testing, NodeRole::Validator)?; - let executor_membership_urls = readiness_urls(executor_testing, NodeRole::Executor)?; + let validator_urls = + readiness_urls(validator_api, NodeRole::Validator, &cluster.validator_host)?; + let executor_urls = readiness_urls(executor_api, NodeRole::Executor, &cluster.executor_host)?; + let validator_membership_urls = readiness_urls( + validator_testing, + NodeRole::Validator, + &cluster.validator_host, + )?; + let executor_membership_urls = + readiness_urls(executor_testing, NodeRole::Executor, &cluster.executor_host)?; descriptors .wait_remote_readiness( @@ -312,7 +340,7 @@ pub async fn wait_for_ports_or_cleanup( { Ok(ports) => { info!( - prometheus_port = ports.ports.prometheus, + prometheus = ?ports.ports.prometheus, validator_ports = ?ports.ports.validators, executor_ports = ?ports.ports.executors, "cluster port-forwards established" @@ -340,36 +368,46 @@ async fn cleanup_pending(client: &Client, namespace: &str, guard: &mut Option Result, RemoteReadinessError> { +fn readiness_urls( + ports: &[u16], + role: NodeRole, + host: &str, +) -> Result, RemoteReadinessError> { ports .iter() .copied() - .map(|port| readiness_url(role, port)) + .map(|port| readiness_url(host, role, port)) .collect() } -fn readiness_url(role: NodeRole, port: u16) -> Result { - cluster_host_url(port).map_err(|source| RemoteReadinessError::Endpoint { role, port, source }) +fn readiness_url(host: &str, role: NodeRole, port: u16) -> Result { + cluster_host_url(host, port).map_err(|source| RemoteReadinessError::Endpoint { + role, + port, + source, + }) } -fn cluster_host_url(port: u16) -> Result { - Url::parse(&format!("http://{}:{port}/", node_host())) +fn cluster_host_url(host: &str, port: u16) -> Result { + Url::parse(&format!("http://{host}:{port}/")) } fn api_client_from_ports( + host: &str, role: NodeRole, api_port: u16, testing_port: u16, ) -> Result { - let base_endpoint = cluster_host_url(api_port).map_err(|source| NodeClientError::Endpoint { - role, - endpoint: "api", - port: api_port, - source, - })?; + let base_endpoint = + cluster_host_url(host, api_port).map_err(|source| NodeClientError::Endpoint { + role, + endpoint: "api", + port: api_port, + source, + })?; let testing_endpoint = Some( - cluster_host_url(testing_port).map_err(|source| NodeClientError::Endpoint { + cluster_host_url(host, testing_port).map_err(|source| NodeClientError::Endpoint { role, endpoint: "testing", port: testing_port, diff --git a/testing-framework/runners/k8s/src/lifecycle/wait/grafana.rs b/testing-framework/runners/k8s/src/lifecycle/wait/grafana.rs new file mode 100644 index 0000000..b0241b1 --- /dev/null +++ b/testing-framework/runners/k8s/src/lifecycle/wait/grafana.rs @@ -0,0 +1,36 @@ +use tokio::time::sleep; + +use super::{ClusterWaitError, node_http_probe_timeout, node_http_timeout}; +use crate::host::node_host; + +const GRAFANA_HTTP_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1); + +pub async fn wait_for_grafana_http_nodeport(port: u16) -> Result<(), ClusterWaitError> { + let host = node_host(); + wait_for_grafana_http(&host, port, node_http_probe_timeout()).await +} + +pub async fn wait_for_grafana_http_port_forward(port: u16) -> Result<(), ClusterWaitError> { + wait_for_grafana_http("127.0.0.1", port, node_http_timeout()).await +} + +async fn wait_for_grafana_http( + host: &str, + port: u16, + timeout: std::time::Duration, +) -> Result<(), ClusterWaitError> { + let client = reqwest::Client::new(); + let url = format!("http://{host}:{port}/api/health"); + + let attempts = timeout.as_secs(); + for _ in 0..attempts { + if let Ok(resp) = client.get(&url).send().await + && resp.status().is_success() + { + return Ok(()); + } + sleep(GRAFANA_HTTP_POLL_INTERVAL).await; + } + + Err(ClusterWaitError::GrafanaTimeout { port }) +} diff --git a/testing-framework/runners/k8s/src/lifecycle/wait/mod.rs b/testing-framework/runners/k8s/src/lifecycle/wait/mod.rs index 89a9d77..51a542c 100644 --- a/testing-framework/runners/k8s/src/lifecycle/wait/mod.rs +++ b/testing-framework/runners/k8s/src/lifecycle/wait/mod.rs @@ -14,6 +14,7 @@ use thiserror::Error; mod deployment; mod forwarding; +mod grafana; mod http_probe; mod orchestrator; mod ports; @@ -36,12 +37,22 @@ pub struct NodePortAllocation { pub testing: u16, } +/// Host/port pair reachable from the machine running the runner. +#[derive(Clone, Debug)] +pub struct HostPort { + pub host: String, + pub port: u16, +} + /// All port assignments for the cluster plus Prometheus. #[derive(Debug)] pub struct ClusterPorts { pub validators: Vec, pub executors: Vec, - pub prometheus: u16, + pub validator_host: String, + pub executor_host: String, + pub prometheus: HostPort, + pub grafana: Option, } /// Success result from waiting for the cluster: host ports and forward handles. @@ -87,6 +98,8 @@ pub enum ClusterWaitError { }, #[error("timeout waiting for prometheus readiness on NodePort {port}")] PrometheusTimeout { port: u16 }, + #[error("timeout waiting for grafana readiness on port {port}")] + GrafanaTimeout { port: u16 }, #[error("failed to start port-forward for service {service} port {port}: {source}")] PortForward { service: String, diff --git a/testing-framework/runners/k8s/src/lifecycle/wait/orchestrator.rs b/testing-framework/runners/k8s/src/lifecycle/wait/orchestrator.rs index 04c7420..db41cb2 100644 --- a/testing-framework/runners/k8s/src/lifecycle/wait/orchestrator.rs +++ b/testing-framework/runners/k8s/src/lifecycle/wait/orchestrator.rs @@ -2,7 +2,7 @@ use kube::Client; use testing_framework_core::scenario::http_probe::NodeRole; use super::{ - ClusterPorts, ClusterReady, ClusterWaitError, NodeConfigPorts, PROMETHEUS_HTTP_PORT, + ClusterPorts, ClusterReady, ClusterWaitError, HostPort, NodeConfigPorts, PROMETHEUS_HTTP_PORT, PROMETHEUS_SERVICE_NAME, prometheus_http_probe_timeout, }; use crate::lifecycle::wait::{ @@ -11,11 +11,14 @@ use crate::lifecycle::wait::{ PortForwardHandle, PortForwardSpawn, kill_port_forwards, port_forward_group, port_forward_service, }, + grafana::{wait_for_grafana_http_nodeport, wait_for_grafana_http_port_forward}, http_probe::{wait_for_node_http_nodeport, wait_for_node_http_port_forward}, ports::{discover_node_ports, find_node_port}, prometheus::{wait_for_prometheus_http_nodeport, wait_for_prometheus_http_port_forward}, }; +const GRAFANA_HTTP_PORT: u16 = 3000; + pub async fn wait_for_cluster_ready( client: &Client, namespace: &str, @@ -28,6 +31,7 @@ pub async fn wait_for_cluster_ready( } let mut validator_allocations = Vec::with_capacity(validator_ports.len()); + let mut validator_host = crate::host::node_host(); for (index, ports) in validator_ports.iter().enumerate() { let name = format!("{release}-validator-{index}"); @@ -47,6 +51,7 @@ pub async fn wait_for_cluster_ready( .is_err() { validator_allocations.clear(); + validator_host = "127.0.0.1".to_owned(); port_forwards = port_forward_group( namespace, release, @@ -67,6 +72,7 @@ pub async fn wait_for_cluster_ready( } let mut executor_allocations = Vec::with_capacity(executor_ports.len()); + let mut executor_host = crate::host::node_host(); for (index, ports) in executor_ports.iter().enumerate() { let name = format!("{release}-executor-{index}"); wait_for_deployment_ready(client, namespace, &name).await?; @@ -81,6 +87,7 @@ pub async fn wait_for_cluster_ready( .is_err() { executor_allocations.clear(); + executor_host = "127.0.0.1".to_owned(); match port_forward_group( namespace, release, @@ -107,6 +114,7 @@ pub async fn wait_for_cluster_ready( PROMETHEUS_HTTP_PORT, ) .await?; + let mut prometheus_host = crate::host::node_host(); if wait_for_prometheus_http_nodeport(prometheus_port, prometheus_http_probe_timeout()) .await .is_err() @@ -118,17 +126,52 @@ pub async fn wait_for_cluster_ready( err })?; prometheus_port = local_port; + prometheus_host = "127.0.0.1".to_owned(); port_forwards.push(handle); if let Err(err) = wait_for_prometheus_http_port_forward(prometheus_port).await { return Err(cleanup_port_forwards(&mut port_forwards, err)); } } + let mut grafana = None; + let grafana_service = format!("{release}-grafana"); + if let Ok(node_port) = + find_node_port(client, namespace, &grafana_service, GRAFANA_HTTP_PORT).await + { + let mut grafana_host = crate::host::node_host(); + let mut grafana_port = node_port; + if wait_for_grafana_http_nodeport(grafana_port).await.is_err() { + let PortForwardSpawn { local_port, handle } = + port_forward_service(namespace, &grafana_service, GRAFANA_HTTP_PORT).map_err( + |err| { + kill_port_forwards(&mut port_forwards); + err + }, + )?; + grafana_host = "127.0.0.1".to_owned(); + grafana_port = local_port; + port_forwards.push(handle); + if let Err(err) = wait_for_grafana_http_port_forward(grafana_port).await { + return Err(cleanup_port_forwards(&mut port_forwards, err)); + } + } + grafana = Some(HostPort { + host: grafana_host, + port: grafana_port, + }); + } + Ok(ClusterReady { ports: ClusterPorts { validators: validator_allocations, executors: executor_allocations, - prometheus: prometheus_port, + validator_host, + executor_host, + prometheus: HostPort { + host: prometheus_host, + port: prometheus_port, + }, + grafana, }, port_forwards, }) diff --git a/versions.env b/versions.env index a066d31..ac50f41 100644 --- a/versions.env +++ b/versions.env @@ -1,7 +1,7 @@ VERSION=v0.3.1 NOMOS_BUNDLE_VERSION=v4 # Pinned nomos-node revision used for CI builds and binary bundles. -NOMOS_NODE_REV=3f623e0c9d159bd8b398a25eda8db0642c491f99 +NOMOS_NODE_REV=ad104981ca79da20183550b5aced9e49773fb6d5 # Optional: local nomos-node checkout override (do not commit absolute paths). # NOMOS_NODE_PATH=