diff --git a/.cargo-deny.toml b/.cargo-deny.toml index d20a51e..f4542dd 100644 --- a/.cargo-deny.toml +++ b/.cargo-deny.toml @@ -29,6 +29,7 @@ allow = [ "BSD-2-Clause", "BSD-3-Clause", "BSL-1.0", + "BlueOak-1.0.0", "CC0-1.0", "CDLA-Permissive-2.0", "ISC", diff --git a/Cargo.lock b/Cargo.lock index 605637e..3605aff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,12 +54,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.100" @@ -874,6 +918,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.24.0" @@ -1174,8 +1224,11 @@ version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", + "terminal_size", ] [[package]] @@ -1209,6 +1262,12 @@ dependencies = [ "owo-colors", ] +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + [[package]] name = "common-http-client" version = "0.1.0" @@ -1235,6 +1294,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e45a4a8926227e4197636ba97a9fc9b00477e9f4bd711395687c5f0734bec4" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.61.2", +] + [[package]] name = "const-hex" version = "1.17.0" @@ -1268,6 +1340,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1427,6 +1508,75 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "cucumber" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18c09939b8de21501b829a3839fa8a01ef6cc226e6bc1f5f163f7104bd5e847d" +dependencies = [ + "anyhow", + "clap", + "console", + "cucumber-codegen", + "cucumber-expressions", + "derive_more", + "either", + "futures", + "gherkin", + "globwalk", + "humantime", + "inventory", + "itertools 0.14.0", + "linked-hash-map", + "pin-project", + "ref-cast", + "regex", + "sealed", + "smart-default", +] + +[[package]] +name = "cucumber-codegen" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f5afe541b5147a7b986816153ccfd502622bb37789420cfff412685f27c0a95" +dependencies = [ + "cucumber-expressions", + "inflections", + "itertools 0.14.0", + "proc-macro2", + "quote", + "regex", + "syn 2.0.111", + "synthez", +] + +[[package]] +name = "cucumber-expressions" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6401038de3af44fe74e6fccdb8a5b7db7ba418f480c8e9ad584c6f65c05a27a6" +dependencies = [ + "derive_more", + "either", + "nom 8.0.0", + "nom_locate", + "regex", + "regex-syntax", +] + +[[package]] +name = "cucumber_ext" +version = "0.1.0" +dependencies = [ + "cucumber", + "testing-framework-core", + "testing-framework-runner-compose", + "testing-framework-runner-local", + "testing-framework-workflows", + "thiserror 2.0.17", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -1602,6 +1752,29 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "derive_more" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" +dependencies = [ + "convert_case 0.10.0", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.111", + "unicode-xid", +] + [[package]] name = "deunicode" version = "1.6.2" @@ -1794,6 +1967,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "enum-as-inner" version = "0.6.1" @@ -2177,6 +2356,23 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gherkin" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70197ce7751bfe8bc828e3a855502d3a869a1e9416b58b10c4bde5cf8a0a3cb3" +dependencies = [ + "heck", + "peg", + "quote", + "serde", + "serde_json", + "syn 2.0.111", + "textwrap", + "thiserror 2.0.17", + "typed-builder", +] + [[package]] name = "gimli" version = "0.32.3" @@ -2947,6 +3143,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inflections" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a257582fdcde896fd96463bf2d40eefea0580021c0712a0e2b028b60b47a837a" + [[package]] name = "inout" version = "0.1.4" @@ -2965,6 +3167,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "inventory" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e" +dependencies = [ + "rustversion", +] + [[package]] name = "ipconfig" version = "0.3.2" @@ -2993,6 +3204,12 @@ dependencies = [ "serde", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.10.5" @@ -3786,6 +4003,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -4212,6 +4435,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "nom_locate" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b577e2d69827c4740cba2b52efaad1c4cc7c73042860b199710b3575c68438d" +dependencies = [ + "bytecount", + "memchr", + "nom 8.0.0", +] + [[package]] name = "nomos-api" version = "0.1.0" @@ -5043,6 +5277,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openssl" version = "0.10.75" @@ -5206,7 +5446,7 @@ name = "overwatch-derive" version = "0.1.0" source = "git+https://github.com/logos-co/Overwatch?rev=f5a9902#f5a99022f389d65adbd55e51f1e3f9eead62432a" dependencies = [ - "convert_case", + "convert_case 0.8.0", "proc-macro-error2", "proc-macro2", "quote", @@ -5263,6 +5503,33 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "peg" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f76678828272f177ac33b7e2ac2e3e73cc6c1cd1e3e387928aa69562fa51367" +dependencies = [ + "peg-macros", + "peg-runtime", +] + +[[package]] +name = "peg-macros" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "636d60acf97633e48d266d7415a9355d4389cea327a193f87df395d88cd2b14d" +dependencies = [ + "peg-runtime", + "proc-macro2", + "quote", +] + +[[package]] +name = "peg-runtime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555b1514d2d99d78150d3c799d4c357a3e2c2a8062cd108e93a06d9057629c5" + [[package]] name = "pem" version = "3.0.6" @@ -5866,6 +6133,26 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "regex" version = "1.12.2" @@ -6032,6 +6319,8 @@ name = "runner-examples" version = "0.1.0" dependencies = [ "anyhow", + "cucumber", + "cucumber_ext", "testing-framework-core", "testing-framework-runner-compose", "testing-framework-runner-k8s", @@ -6269,6 +6558,17 @@ version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" +[[package]] +name = "sealed" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f968c5ea23d555e670b449c1c5e7b2fc399fdaec1d304a17cd48e288abc107" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "sec1" version = "0.7.3" @@ -6576,6 +6876,23 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "smart-default" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "smawk" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c" + [[package]] name = "snap" version = "1.1.1" @@ -6731,6 +7048,39 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "synthez" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d8a928f38f1bc873f28e0d2ba8298ad65374a6ac2241dabd297271531a736cd" +dependencies = [ + "syn 2.0.111", + "synthez-codegen", + "synthez-core", +] + +[[package]] +name = "synthez-codegen" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb83b8df4238e11746984dfb3819b155cd270de0e25847f45abad56b3671047" +dependencies = [ + "syn 2.0.111", + "synthez-core", +] + +[[package]] +name = "synthez-core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906fba967105d822e7c7ed60477b5e76116724d33de68a585681fb253fc30d5c" +dependencies = [ + "proc-macro2", + "quote", + "sealed", + "syn 2.0.111", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -6820,6 +7170,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "terminal_size" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" +dependencies = [ + "rustix", + "windows-sys 0.60.2", +] + [[package]] name = "testing-framework-config" version = "0.1.0" @@ -7026,6 +7386,17 @@ dependencies = [ "tx-service", ] +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -7618,6 +7989,26 @@ dependencies = [ "utoipa", ] +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "typenum" version = "1.19.0" @@ -7660,12 +8051,30 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-segmentation" version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -7708,6 +8117,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "utoipa" version = "4.2.3" diff --git a/Cargo.toml b/Cargo.toml index a6a5be3..00cb219 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "examples/doc-snippets", "testing-framework/configs", "testing-framework/core", + "testing-framework/cucumber_ext", "testing-framework/deployers/compose", "testing-framework/deployers/k8s", "testing-framework/deployers/local", diff --git a/examples/Cargo.toml b/examples/Cargo.toml index ba28e95..24c80dd 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -11,6 +11,8 @@ version = "0.1.0" [dependencies] anyhow = "1" +cucumber = { version = "0.22.0" } +cucumber_ext = { path = "../testing-framework/cucumber_ext" } testing-framework-core = { workspace = true } testing-framework-runner-compose = { workspace = true } testing-framework-runner-k8s = { workspace = true } diff --git a/examples/cucumber/features/compose_smoke.feature b/examples/cucumber/features/compose_smoke.feature new file mode 100644 index 0000000..d0663e4 --- /dev/null +++ b/examples/cucumber/features/compose_smoke.feature @@ -0,0 +1,13 @@ +@compose +Feature: Testing Framework - Compose Runner + + Scenario: Run a compose smoke scenario (tx + DA + liveness) + Given deployer is "compose" + And topology has 1 validators and 1 executors + And wallets total funds is 1000 split across 10 users + And run duration is 60 seconds + And transactions rate is 1 per block + And data availability channel rate is 1 per block and blob rate is 1 per block + And expect consensus liveness + When run scenario + Then scenario should succeed diff --git a/examples/cucumber/features/local_smoke.feature b/examples/cucumber/features/local_smoke.feature new file mode 100644 index 0000000..ef28ca5 --- /dev/null +++ b/examples/cucumber/features/local_smoke.feature @@ -0,0 +1,14 @@ +Feature: Testing Framework - Local Runner + + @local + Scenario: Run a local smoke scenario (tx + DA + liveness) + Given deployer is "local" + And topology has 1 validators and 1 executors + And run duration is 60 seconds + And wallets total funds is 1000000000 split across 50 users + And transactions rate is 1 per block + And data availability channel rate is 1 per block and blob rate is 1 per block + And expect consensus liveness + When run scenario + Then scenario should succeed + diff --git a/examples/src/bin/cucumber_compose.rs b/examples/src/bin/cucumber_compose.rs new file mode 100644 index 0000000..263f143 --- /dev/null +++ b/examples/src/bin/cucumber_compose.rs @@ -0,0 +1,9 @@ +use runner_examples::cucumber::{Mode, init_logging_defaults, init_tracing, run}; + +#[tokio::main(flavor = "current_thread")] +async fn main() { + init_logging_defaults(); + init_tracing(); + + run(Mode::Compose).await; +} diff --git a/examples/src/bin/cucumber_host.rs b/examples/src/bin/cucumber_host.rs new file mode 100644 index 0000000..7fb02bc --- /dev/null +++ b/examples/src/bin/cucumber_host.rs @@ -0,0 +1,8 @@ +use runner_examples::cucumber::{Mode, init_logging_defaults, init_tracing, run}; + +#[tokio::main(flavor = "current_thread")] +async fn main() { + init_logging_defaults(); + init_tracing(); + run(Mode::Host).await; +} diff --git a/examples/src/cucumber.rs b/examples/src/cucumber.rs new file mode 100644 index 0000000..0eeb07e --- /dev/null +++ b/examples/src/cucumber.rs @@ -0,0 +1,53 @@ +use cucumber::World; +use cucumber_ext::TestingFrameworkWorld; +use tracing_subscriber::{EnvFilter, fmt}; + +const FEATURES_PATH: &str = "examples/cucumber/features"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Mode { + Host, + Compose, +} + +fn set_default_env(key: &str, value: &str) { + if std::env::var_os(key).is_none() { + // SAFETY: Used as an early-run default. Prefer setting env vars in the + // shell for multi-threaded runs. + unsafe { + std::env::set_var(key, value); + } + } +} + +fn is_compose( + feature: &cucumber::gherkin::Feature, + scenario: &cucumber::gherkin::Scenario, +) -> bool { + scenario.tags.iter().any(|tag| tag == "compose") + || feature.tags.iter().any(|tag| tag == "compose") +} + +pub fn init_logging_defaults() { + set_default_env("POL_PROOF_DEV_MODE", "true"); + set_default_env("NOMOS_TESTS_KEEP_LOGS", "1"); + set_default_env("NOMOS_LOG_DIR", ".tmp/cucumber-logs"); + set_default_env("NOMOS_LOG_LEVEL", "info"); + set_default_env("RUST_LOG", "info"); +} + +pub fn init_tracing() { + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + let _ = fmt().with_env_filter(filter).with_target(true).try_init(); +} + +pub async fn run(mode: Mode) { + TestingFrameworkWorld::cucumber() + .with_default_cli() + .max_concurrent_scenarios(Some(1)) + .filter_run(FEATURES_PATH, move |feature, _, scenario| match mode { + Mode::Host => !is_compose(feature, scenario), + Mode::Compose => is_compose(feature, scenario), + }) + .await; +} diff --git a/examples/src/lib.rs b/examples/src/lib.rs index bf0bcae..c3186b3 100644 --- a/examples/src/lib.rs +++ b/examples/src/lib.rs @@ -1,15 +1,5 @@ -use testing_framework_core::scenario::Metrics; -pub use testing_framework_workflows::{ - builder::{ChaosBuilderExt, ScenarioBuilderExt}, - expectations, util, workloads, -}; - +pub mod cucumber; pub mod env; pub use env::read_env_any; - -/// Metrics are currently disabled in this branch; return a stub handle. -#[must_use] -pub const fn configure_prometheus_metrics() -> Metrics { - Metrics::empty() -} +pub use testing_framework_workflows::{ChaosBuilderExt, ScenarioBuilderExt}; diff --git a/testing-framework/cucumber_ext/Cargo.toml b/testing-framework/cucumber_ext/Cargo.toml new file mode 100644 index 0000000..d7ec8f3 --- /dev/null +++ b/testing-framework/cucumber_ext/Cargo.toml @@ -0,0 +1,21 @@ +[package] +categories.workspace = true +description.workspace = true +edition.workspace = true +keywords.workspace = true +license.workspace = true +name = "cucumber_ext" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[dependencies] +cucumber = { version = "0.22.0", features = ["default", "macros"] } +testing-framework-core = { workspace = true } +testing-framework-runner-compose = { workspace = true } +testing-framework-runner-local = { workspace = true } +testing-framework-workflows = { workspace = true } +thiserror = { workspace = true } + +[lints] +workspace = true diff --git a/testing-framework/cucumber_ext/src/lib.rs b/testing-framework/cucumber_ext/src/lib.rs new file mode 100644 index 0000000..48bf550 --- /dev/null +++ b/testing-framework/cucumber_ext/src/lib.rs @@ -0,0 +1,4 @@ +mod steps; +mod world; + +pub use world::TestingFrameworkWorld; diff --git a/testing-framework/cucumber_ext/src/steps/mod.rs b/testing-framework/cucumber_ext/src/steps/mod.rs new file mode 100644 index 0000000..8c3581a --- /dev/null +++ b/testing-framework/cucumber_ext/src/steps/mod.rs @@ -0,0 +1,3 @@ +mod run; +mod scenario; +mod workloads; diff --git a/testing-framework/cucumber_ext/src/steps/run.rs b/testing-framework/cucumber_ext/src/steps/run.rs new file mode 100644 index 0000000..888039b --- /dev/null +++ b/testing-framework/cucumber_ext/src/steps/run.rs @@ -0,0 +1,72 @@ +use cucumber::{then, when}; +use testing_framework_core::scenario::Deployer as _; +use testing_framework_runner_compose::ComposeDeployer; +use testing_framework_runner_local::LocalDeployer; + +use crate::world::{DeployerKind, StepError, StepResult, TestingFrameworkWorld}; + +#[when(expr = "run scenario")] +async fn run_scenario(world: &mut TestingFrameworkWorld) -> StepResult { + let deployer = world.deployer.ok_or(StepError::MissingDeployer)?; + world.run.result = Some(match deployer { + DeployerKind::Local => { + let mut scenario = world.build_local_scenario()?; + let deployer = LocalDeployer::default().with_membership_check(world.membership_check); + let result = async { + let runner = + deployer + .deploy(&scenario) + .await + .map_err(|e| StepError::RunFailed { + message: format!("local deploy failed: {e}"), + })?; + runner + .run(&mut scenario) + .await + .map_err(|e| StepError::RunFailed { + message: format!("scenario run failed: {e}"), + })?; + Ok::<(), StepError>(()) + } + .await; + + result.map_err(|e| e.to_string()) + } + DeployerKind::Compose => { + let mut scenario = world.build_compose_scenario()?; + let deployer = ComposeDeployer::default().with_readiness(world.readiness_checks); + let result = async { + let runner = + deployer + .deploy(&scenario) + .await + .map_err(|e| StepError::RunFailed { + message: format!("compose deploy failed: {e}"), + })?; + runner + .run(&mut scenario) + .await + .map_err(|e| StepError::RunFailed { + message: format!("scenario run failed: {e}"), + })?; + Ok::<(), StepError>(()) + } + .await; + + result.map_err(|e| e.to_string()) + } + }); + + Ok(()) +} + +#[then(expr = "scenario should succeed")] +async fn scenario_should_succeed(world: &mut TestingFrameworkWorld) -> StepResult { + match world.run.result.take() { + Some(Ok(())) => Ok(()), + Some(Err(message)) => Err(StepError::RunFailed { message }), + None => Err(StepError::RunFailed { + message: "scenario was not run".to_owned(), + }), + } +} diff --git a/testing-framework/cucumber_ext/src/steps/scenario.rs b/testing-framework/cucumber_ext/src/steps/scenario.rs new file mode 100644 index 0000000..1adccef --- /dev/null +++ b/testing-framework/cucumber_ext/src/steps/scenario.rs @@ -0,0 +1,17 @@ +use cucumber::given; + +use crate::world::{NetworkKind, StepResult, TestingFrameworkWorld, parse_deployer}; + +#[given(expr = "deployer is {string}")] +async fn deployer_is(world: &mut TestingFrameworkWorld, deployer: String) -> StepResult { + world.set_deployer(parse_deployer(&deployer)?) +} + +#[given(expr = "topology has {int} validators and {int} executors")] +async fn topology_has( + world: &mut TestingFrameworkWorld, + validators: usize, + executors: usize, +) -> StepResult { + world.set_topology(validators, executors, NetworkKind::Star) +} diff --git a/testing-framework/cucumber_ext/src/steps/workloads.rs b/testing-framework/cucumber_ext/src/steps/workloads.rs new file mode 100644 index 0000000..af2672c --- /dev/null +++ b/testing-framework/cucumber_ext/src/steps/workloads.rs @@ -0,0 +1,52 @@ +use cucumber::given; + +use crate::world::{StepResult, TestingFrameworkWorld}; + +#[given(expr = "wallets total funds is {int} split across {int} users")] +async fn wallets_total_funds( + world: &mut TestingFrameworkWorld, + total_funds: u64, + users: usize, +) -> StepResult { + world.set_wallets(total_funds, users) +} + +#[given(expr = "run duration is {int} seconds")] +async fn run_duration(world: &mut TestingFrameworkWorld, seconds: u64) -> StepResult { + world.set_run_duration(seconds) +} + +#[given(expr = "transactions rate is {int} per block")] +async fn tx_rate(world: &mut TestingFrameworkWorld, rate: u64) -> StepResult { + world.set_transactions_rate(rate, None) +} + +#[given(expr = "transactions rate is {int} per block using {int} users")] +async fn tx_rate_with_users( + world: &mut TestingFrameworkWorld, + rate: u64, + users: usize, +) -> StepResult { + world.set_transactions_rate(rate, Some(users)) +} + +#[given( + expr = "data availability channel rate is {int} per block and blob rate is {int} per block" +)] +async fn da_rates( + world: &mut TestingFrameworkWorld, + channel_rate: u64, + blob_rate: u64, +) -> StepResult { + world.set_data_availability_rates(channel_rate, blob_rate) +} + +#[given(expr = "expect consensus liveness")] +async fn expect_consensus_liveness(world: &mut TestingFrameworkWorld) -> StepResult { + world.enable_consensus_liveness() +} + +#[given(expr = "consensus liveness lag allowance is {int}")] +async fn liveness_lag_allowance(world: &mut TestingFrameworkWorld, blocks: u64) -> StepResult { + world.set_consensus_liveness_lag_allowance(blocks) +} diff --git a/testing-framework/cucumber_ext/src/world.rs b/testing-framework/cucumber_ext/src/world.rs new file mode 100644 index 0000000..a87c779 --- /dev/null +++ b/testing-framework/cucumber_ext/src/world.rs @@ -0,0 +1,345 @@ +use std::{env, path::PathBuf, time::Duration}; + +use cucumber::World; +use testing_framework_core::scenario::{Builder, NodeControlCapability, Scenario, ScenarioBuilder}; +use testing_framework_workflows::{ScenarioBuilderExt as _, expectations::ConsensusLiveness}; +use thiserror::Error; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum DeployerKind { + #[default] + Local, + Compose, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NetworkKind { + Star, +} + +#[derive(Debug, Default, Clone)] +pub struct RunState { + pub result: Option>, +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct ScenarioSpec { + pub topology: Option, + pub duration_secs: Option, + pub wallets: Option, + pub transactions: Option, + pub data_availability: Option, + pub consensus_liveness: Option, +} + +#[derive(Debug, Clone, Copy)] +pub struct TopologySpec { + pub validators: usize, + pub executors: usize, + pub network: NetworkKind, +} + +#[derive(Debug, Clone, Copy)] +pub struct WalletSpec { + pub total_funds: u64, + pub users: usize, +} + +#[derive(Debug, Clone, Copy)] +pub struct TransactionSpec { + pub rate_per_block: u64, + pub users: Option, +} + +#[derive(Debug, Clone, Copy)] +pub struct DataAvailabilitySpec { + pub channel_rate_per_block: u64, + pub blob_rate_per_block: u64, +} + +#[derive(Debug, Clone, Copy)] +pub struct ConsensusLivenessSpec { + pub lag_allowance: Option, +} + +#[derive(Debug, Error)] +pub enum StepError { + #[error("deployer is not selected; set it first (e.g. `Given deployer is \"local\"`)")] + MissingDeployer, + #[error("scenario topology is not configured")] + MissingTopology, + #[error("scenario run duration is not configured")] + MissingRunDuration, + #[error("unsupported deployer kind: {value}")] + UnsupportedDeployer { value: String }, + #[error("step requires deployer {expected:?}, but current deployer is {actual:?}")] + DeployerMismatch { + expected: DeployerKind, + actual: DeployerKind, + }, + #[error("invalid argument: {message}")] + InvalidArgument { message: String }, + #[error("{message}")] + Preflight { message: String }, + #[error("{message}")] + RunFailed { message: String }, +} + +pub type StepResult = Result<(), StepError>; + +#[derive(World, Debug, Default)] +pub struct TestingFrameworkWorld { + pub deployer: Option, + pub spec: ScenarioSpec, + pub run: RunState, + pub membership_check: bool, + pub readiness_checks: bool, +} + +impl TestingFrameworkWorld { + pub fn set_deployer(&mut self, kind: DeployerKind) -> StepResult { + self.deployer = Some(kind); + Ok(()) + } + + pub fn set_topology( + &mut self, + validators: usize, + executors: usize, + network: NetworkKind, + ) -> StepResult { + self.spec.topology = Some(TopologySpec { + validators: positive_usize("validators", validators)?, + executors: positive_usize("executors", executors)?, + network, + }); + Ok(()) + } + + pub fn set_run_duration(&mut self, seconds: u64) -> StepResult { + self.spec.duration_secs = Some(positive_u64("duration", seconds)?); + Ok(()) + } + + pub fn set_wallets(&mut self, total_funds: u64, users: usize) -> StepResult { + self.spec.wallets = Some(WalletSpec { + total_funds, + users: positive_usize("wallet users", users)?, + }); + Ok(()) + } + + pub fn set_transactions_rate( + &mut self, + rate_per_block: u64, + users: Option, + ) -> StepResult { + if self.spec.transactions.is_some() { + return Err(StepError::InvalidArgument { + message: "transactions workload already configured".to_owned(), + }); + } + + if users.is_some_and(|u| u == 0) { + return Err(StepError::InvalidArgument { + message: "transactions users must be > 0".to_owned(), + }); + } + + self.spec.transactions = Some(TransactionSpec { + rate_per_block: positive_u64("transactions rate", rate_per_block)?, + users, + }); + Ok(()) + } + + pub fn set_data_availability_rates( + &mut self, + channel_rate_per_block: u64, + blob_rate_per_block: u64, + ) -> StepResult { + if self.spec.data_availability.is_some() { + return Err(StepError::InvalidArgument { + message: "data availability workload already configured".to_owned(), + }); + } + + self.spec.data_availability = Some(DataAvailabilitySpec { + channel_rate_per_block: positive_u64("DA channel rate", channel_rate_per_block)?, + blob_rate_per_block: positive_u64("DA blob rate", blob_rate_per_block)?, + }); + + Ok(()) + } + + pub fn enable_consensus_liveness(&mut self) -> StepResult { + if self.spec.consensus_liveness.is_none() { + self.spec.consensus_liveness = Some(ConsensusLivenessSpec { + lag_allowance: None, + }); + } + + Ok(()) + } + + pub fn set_consensus_liveness_lag_allowance(&mut self, blocks: u64) -> StepResult { + let blocks = positive_u64("lag allowance", blocks)?; + + self.spec.consensus_liveness = Some(ConsensusLivenessSpec { + lag_allowance: Some(blocks), + }); + + Ok(()) + } + + pub fn build_local_scenario(&self) -> Result, StepError> { + self.preflight(DeployerKind::Local)?; + let builder = self.make_builder_for_deployer::<()>(DeployerKind::Local)?; + Ok(builder.build()) + } + + pub fn build_compose_scenario(&self) -> Result, StepError> { + self.preflight(DeployerKind::Compose)?; + let builder = + self.make_builder_for_deployer::(DeployerKind::Compose)?; + Ok(builder.build()) + } + + pub fn preflight(&self, expected: DeployerKind) -> Result<(), StepError> { + let actual = self.deployer.ok_or(StepError::MissingDeployer)?; + if actual != expected { + return Err(StepError::DeployerMismatch { expected, actual }); + } + + if !is_truthy_env("POL_PROOF_DEV_MODE") { + return Err(StepError::Preflight { + message: + "POL_PROOF_DEV_MODE must be set to \"true\" (or \"1\") for practical test runs." + .to_owned(), + }); + } + + if expected == DeployerKind::Local { + let node_ok = env::var_os("NOMOS_NODE_BIN") + .map(PathBuf::from) + .is_some_and(|p| p.is_file()) + || shared_host_bin_path("nomos-node").is_file(); + + let exec_ok = env::var_os("NOMOS_EXECUTOR_BIN") + .map(PathBuf::from) + .is_some_and(|p| p.is_file()) + || shared_host_bin_path("nomos-executor").is_file(); + + if !(node_ok && exec_ok) { + return Err(StepError::Preflight { + message: "Missing Nomos host binaries. Set NOMOS_NODE_BIN and NOMOS_EXECUTOR_BIN, or run `scripts/run-examples.sh host` to restore them into `testing-framework/assets/stack/bin`.".to_owned(), + }); + } + } + + Ok(()) + } + + fn make_builder_for_deployer( + &self, + expected: DeployerKind, + ) -> Result, StepError> { + let actual = self.deployer.ok_or(StepError::MissingDeployer)?; + if actual != expected { + return Err(StepError::DeployerMismatch { expected, actual }); + } + + let topology = self.spec.topology.ok_or(StepError::MissingTopology)?; + let duration_secs = self + .spec + .duration_secs + .ok_or(StepError::MissingRunDuration)?; + + let mut builder: Builder = make_builder(topology).with_capabilities(Caps::default()); + + builder = builder.with_run_duration(Duration::from_secs(duration_secs)); + + if let Some(wallets) = self.spec.wallets { + builder = builder.initialize_wallet(wallets.total_funds, wallets.users); + } + + if let Some(tx) = self.spec.transactions { + builder = builder.transactions_with(|flow| { + let mut flow = flow.rate(tx.rate_per_block); + if let Some(users) = tx.users { + flow = flow.users(users); + } + flow + }); + } + + if let Some(da) = self.spec.data_availability { + builder = builder.da_with(|flow| { + flow.channel_rate(da.channel_rate_per_block) + .blob_rate(da.blob_rate_per_block) + }); + } + + if let Some(liveness) = self.spec.consensus_liveness { + if let Some(lag) = liveness.lag_allowance { + builder = + builder.with_expectation(ConsensusLiveness::default().with_lag_allowance(lag)); + } else { + builder = builder.expect_consensus_liveness(); + } + } + + Ok(builder) + } +} + +fn make_builder(topology: TopologySpec) -> Builder<()> { + ScenarioBuilder::topology_with(|t| { + let base = match topology.network { + NetworkKind::Star => t.network_star(), + }; + base.validators(topology.validators) + .executors(topology.executors) + }) +} + +fn is_truthy_env(key: &str) -> bool { + env::var(key) + .ok() + .is_some_and(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) +} + +fn positive_usize(label: &str, value: usize) -> Result { + if value == 0 { + Err(StepError::InvalidArgument { + message: format!("{label} must be > 0"), + }) + } else { + Ok(value) + } +} + +fn positive_u64(label: &str, value: u64) -> Result { + if value == 0 { + Err(StepError::InvalidArgument { + message: format!("{label} must be > 0"), + }) + } else { + Ok(value) + } +} + +pub fn parse_deployer(value: &str) -> Result { + match value.trim().to_ascii_lowercase().as_str() { + "local" | "host" => Ok(DeployerKind::Local), + "compose" | "docker" => Ok(DeployerKind::Compose), + other => Err(StepError::UnsupportedDeployer { + value: other.to_owned(), + }), + } +} + +pub fn shared_host_bin_path(binary_name: &str) -> PathBuf { + let cucumber_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + cucumber_dir.join("../assets/stack/bin").join(binary_name) +}