diff --git a/Cargo.lock b/Cargo.lock index a98949a..36511ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1528,6 +1528,7 @@ dependencies = [ "humantime", "inventory", "itertools 0.14.0", + "junit-report", "linked-hash-map", "pin-project", "ref-cast", @@ -1742,6 +1743,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive-getters" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2c35ab6e03642397cdda1dd58abbc05d418aef8e36297f336d5aba060fe8df" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -3322,6 +3334,18 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "junit-report" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c3a3342e6720a82d7d179f380e9841b73a1dd49344e33959fdfe571ce56b55" +dependencies = [ + "derive-getters", + "quick-xml", + "strip-ansi-escapes", + "time", +] + [[package]] name = "k256" version = "0.13.4" @@ -5947,6 +5971,15 @@ dependencies = [ "unsigned-varint 0.8.0", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6961,6 +6994,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -8226,6 +8268,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "walkdir" version = "2.5.0" diff --git a/examples/cucumber/features/auto_deployer_smoke.feature b/examples/cucumber/features/auto_deployer_smoke.feature new file mode 100644 index 0000000..e02cff6 --- /dev/null +++ b/examples/cucumber/features/auto_deployer_smoke.feature @@ -0,0 +1,34 @@ +Feature: Testing Framework - Auto Local/Compose Deployer + + Scenario: Run auto deployer smoke scenario (tx + DA + liveness) + Given we have a CLI deployer specified + And topology has 1 validators and 1 executors + And run duration is 60 seconds + And wallets total funds is 1000000000 split across 50 users + And transactions rate is 1 per block + And data availability channel rate is 1 per block and blob rate is 1 per block + And expect consensus liveness + When run scenario + Then scenario should succeed + + # Note: This test may fail on slow computers + Scenario: Run auto deployer stress smoke scenario (tx + DA + liveness) + Given we have a CLI deployer specified + And topology has 3 validators and 3 executors + And run duration is 120 seconds + And wallets total funds is 1000000000 split across 500 users + And transactions rate is 10 per block + And data availability channel rate is 1 per block and blob rate is 1 per block + And expect consensus liveness + When run scenario + Then scenario should succeed + + Scenario: Run auto deployer stress smoke scenario no liveness (tx + DA + liveness) + Given we have a CLI deployer specified + And topology has 3 validators and 3 executors + And run duration is 120 seconds + And wallets total funds is 1000000000 split across 500 users + And transactions rate is 10 per block + And data availability channel rate is 1 per block and blob rate is 1 per block + When run scenario + Then scenario should succeed diff --git a/examples/cucumber/features/local_smoke.feature b/examples/cucumber/features/local_smoke.feature index ef28ca5..f66630b 100644 --- a/examples/cucumber/features/local_smoke.feature +++ b/examples/cucumber/features/local_smoke.feature @@ -1,6 +1,6 @@ +@local Feature: Testing Framework - Local Runner - @local Scenario: Run a local smoke scenario (tx + DA + liveness) Given deployer is "local" And topology has 1 validators and 1 executors diff --git a/examples/src/bin/compose_runner.rs b/examples/src/bin/compose_runner.rs index 32d2ea1..e11365d 100644 --- a/examples/src/bin/compose_runner.rs +++ b/examples/src/bin/compose_runner.rs @@ -1,9 +1,8 @@ use std::{process, time::Duration}; use anyhow::{Context as _, Result}; -use runner_examples::{ - ChaosBuilderExt as _, ScenarioBuilderExt as _, defaults::Mode, demo, read_env_any, -}; +use cucumber_ext::DeployerKind; +use runner_examples::{ChaosBuilderExt as _, ScenarioBuilderExt as _, demo, read_env_any}; use testing_framework_core::scenario::{Deployer as _, Runner, ScenarioBuilder}; use testing_framework_runner_compose::{ComposeDeployer, ComposeRunnerError}; use tracing::{info, warn}; @@ -24,7 +23,7 @@ const DA_BLOB_RATE: u64 = 1; #[tokio::main] async fn main() { - runner_examples::defaults::init_node_log_dir_defaults(Mode::Compose); + runner_examples::defaults::init_node_log_dir_defaults(DeployerKind::Compose); tracing_subscriber::fmt::init(); diff --git a/examples/src/bin/cucumber_auto.rs b/examples/src/bin/cucumber_auto.rs new file mode 100644 index 0000000..29caaec --- /dev/null +++ b/examples/src/bin/cucumber_auto.rs @@ -0,0 +1,66 @@ +/// Usage: Set the environment variable CUCUMBER_DEPLOYER_COMPOSE to use the +/// Compose deployer. Otherwise, the Local deployer is used by default. +/// +/// Example using docker compose deployer: +/// ```sh +/// CUCUMBER_DEPLOYER_COMPOSE=1 cargo run -p runner-examples --bin cucumber_auto -- --name "Run auto deployer smoke scenario" +/// ``` +/// Example using local deployer: +/// ```sh +/// cargo run -p runner-examples --bin cucumber_auto -- --name "Run auto deployer smoke scenario" +/// ``` +use std::{fs, io}; + +use cucumber::{World, WriterExt, writer, writer::Verbosity}; +use cucumber_ext::{DeployerKind, TestingFrameworkWorld}; +use runner_examples::defaults::{init_logging_defaults, init_node_log_dir_defaults, init_tracing}; + +#[tokio::main] +async fn main() { + println!("args: {:?}", std::env::args()); + + let deployer = if std::env::var("CUCUMBER_DEPLOYER_COMPOSE").ok().is_some() { + DeployerKind::Compose + } else { + DeployerKind::Local + }; + println!("Running with '{:?}'", deployer); + + init_logging_defaults(); + init_node_log_dir_defaults(deployer); + init_tracing(); + + // Print current directory for debugging + if let Ok(current_dir) = std::env::current_dir() { + println!("Current directory: {:?}", current_dir); + } + + let file = fs::File::create("cucumber-output-junit.xml").unwrap(); + let world = TestingFrameworkWorld::cucumber() + .repeat_failed() + // following config needed to use eprint statements in the tests + .max_concurrent_scenarios(1) + .fail_on_skipped() + .fail_fast() + .with_writer( + writer::Summarize::new(writer::Basic::new( + io::stdout(), + writer::Coloring::Auto, + Verbosity::ShowWorldAndDocString, + )) + .tee::(writer::JUnit::for_tee(file, 0)) + .normalized(), + ) + .before(move |feature, _rule, scenario, world| { + Box::pin(async move { + println!( + "\nStarting '{}' : '{}' : '{}'\n", + feature.name, scenario.keyword, scenario.name + ); // This will be printed into the stdout_buffer + if let Err(e) = world.set_deployer(deployer) { + panic!("Failed to set deployer: {}", e); + } + }) + }); + world.run_and_exit("examples/cucumber/features/").await; +} diff --git a/examples/src/bin/cucumber_compose.rs b/examples/src/bin/cucumber_compose.rs index fdf079a..30a1939 100644 --- a/examples/src/bin/cucumber_compose.rs +++ b/examples/src/bin/cucumber_compose.rs @@ -1,13 +1,11 @@ use cucumber::World; -use cucumber_ext::TestingFrameworkWorld; -use runner_examples::defaults::{ - Mode, init_logging_defaults, init_node_log_dir_defaults, init_tracing, -}; +use cucumber_ext::{DeployerKind, TestingFrameworkWorld}; +use runner_examples::defaults::{init_logging_defaults, init_node_log_dir_defaults, init_tracing}; #[tokio::main] async fn main() { init_logging_defaults(); - init_node_log_dir_defaults(Mode::Compose); + init_node_log_dir_defaults(DeployerKind::Compose); init_tracing(); TestingFrameworkWorld::run("examples/cucumber/features/compose_smoke.feature").await; diff --git a/examples/src/bin/cucumber_host.rs b/examples/src/bin/cucumber_host.rs index 074dc7e..e00d353 100644 --- a/examples/src/bin/cucumber_host.rs +++ b/examples/src/bin/cucumber_host.rs @@ -1,13 +1,11 @@ use cucumber::World; -use cucumber_ext::TestingFrameworkWorld; -use runner_examples::defaults::{ - Mode, init_logging_defaults, init_node_log_dir_defaults, init_tracing, -}; +use cucumber_ext::{DeployerKind, TestingFrameworkWorld}; +use runner_examples::defaults::{init_logging_defaults, init_node_log_dir_defaults, init_tracing}; #[tokio::main] async fn main() { init_logging_defaults(); - init_node_log_dir_defaults(Mode::Host); + init_node_log_dir_defaults(DeployerKind::Local); init_tracing(); TestingFrameworkWorld::run("examples/cucumber/features/local_smoke.feature").await; diff --git a/examples/src/bin/local_runner.rs b/examples/src/bin/local_runner.rs index 1565106..8caca26 100644 --- a/examples/src/bin/local_runner.rs +++ b/examples/src/bin/local_runner.rs @@ -1,7 +1,8 @@ use std::{env, process, time::Duration}; use anyhow::{Context as _, Result}; -use runner_examples::{ScenarioBuilderExt as _, defaults::Mode, demo, read_env_any}; +use cucumber_ext::DeployerKind; +use runner_examples::{ScenarioBuilderExt as _, demo, read_env_any}; use testing_framework_core::scenario::{Deployer as _, Runner, ScenarioBuilder}; use testing_framework_runner_local::LocalDeployer; use tracing::{info, warn}; @@ -14,7 +15,7 @@ const SMOKE_RUN_SECS_MAX: u64 = 30; #[tokio::main] async fn main() { - runner_examples::defaults::init_node_log_dir_defaults(Mode::Host); + runner_examples::defaults::init_node_log_dir_defaults(DeployerKind::Local); tracing_subscriber::fmt::init(); diff --git a/examples/src/cucumber.rs b/examples/src/cucumber.rs deleted file mode 100644 index 7627762..0000000 --- a/examples/src/cucumber.rs +++ /dev/null @@ -1,27 +0,0 @@ -use cucumber::World; -use cucumber_ext::TestingFrameworkWorld; - -pub use crate::defaults::Mode; - -const FEATURES_PATH: &str = "examples/cucumber/features"; - -pub use crate::defaults::{init_logging_defaults, init_node_log_dir_defaults, init_tracing}; - -fn is_compose( - feature: &cucumber::gherkin::Feature, - scenario: &cucumber::gherkin::Scenario, -) -> bool { - scenario.tags.iter().any(|tag| tag == "compose") - || feature.tags.iter().any(|tag| tag == "compose") -} - -pub async fn run(mode: Mode) { - TestingFrameworkWorld::cucumber() - .with_default_cli() - .max_concurrent_scenarios(Some(1)) - .filter_run(FEATURES_PATH, move |feature, _, scenario| match mode { - Mode::Host => !is_compose(feature, scenario), - Mode::Compose => is_compose(feature, scenario), - }) - .await; -} diff --git a/examples/src/defaults.rs b/examples/src/defaults.rs index b1e6c8e..10c03cd 100644 --- a/examples/src/defaults.rs +++ b/examples/src/defaults.rs @@ -3,14 +3,9 @@ use std::{ path::{Path, PathBuf}, }; +use cucumber_ext::DeployerKind; use tracing_subscriber::{EnvFilter, fmt}; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum Mode { - Host, - Compose, -} - const DEFAULT_NODE_LOG_DIR_REL: &str = ".tmp/node-logs"; const DEFAULT_CONTAINER_NODE_LOG_DIR: &str = "/tmp/node-logs"; @@ -31,7 +26,7 @@ pub fn init_logging_defaults() { set_default_env("RUST_LOG", "info"); } -pub fn init_node_log_dir_defaults(mode: Mode) { +pub fn init_node_log_dir_defaults(deployer: DeployerKind) { if env::var_os("NOMOS_LOG_DIR").is_some() { return; } @@ -39,9 +34,9 @@ pub fn init_node_log_dir_defaults(mode: Mode) { let host_dir = repo_root().join(DEFAULT_NODE_LOG_DIR_REL); let _ = fs::create_dir_all(&host_dir); - match mode { - Mode::Host => set_default_env("NOMOS_LOG_DIR", &host_dir.display().to_string()), - Mode::Compose => set_default_env("NOMOS_LOG_DIR", DEFAULT_CONTAINER_NODE_LOG_DIR), + match deployer { + DeployerKind::Local => set_default_env("NOMOS_LOG_DIR", &host_dir.display().to_string()), + DeployerKind::Compose => set_default_env("NOMOS_LOG_DIR", DEFAULT_CONTAINER_NODE_LOG_DIR), } } diff --git a/examples/src/lib.rs b/examples/src/lib.rs index a124fc3..5792838 100644 --- a/examples/src/lib.rs +++ b/examples/src/lib.rs @@ -1,4 +1,3 @@ -pub mod cucumber; pub mod defaults; pub mod demo; pub mod env; diff --git a/testing-framework/cucumber/Cargo.toml b/testing-framework/cucumber/Cargo.toml index d7ec8f3..de7cef4 100644 --- a/testing-framework/cucumber/Cargo.toml +++ b/testing-framework/cucumber/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true version.workspace = true [dependencies] -cucumber = { version = "0.22.0", features = ["default", "macros"] } +cucumber = { version = "0.22.0", features = ["default", "macros", "output-junit"] } testing-framework-core = { workspace = true } testing-framework-runner-compose = { workspace = true } testing-framework-runner-local = { workspace = true } diff --git a/testing-framework/cucumber/src/lib.rs b/testing-framework/cucumber/src/lib.rs index 48bf550..7ddfc6e 100644 --- a/testing-framework/cucumber/src/lib.rs +++ b/testing-framework/cucumber/src/lib.rs @@ -1,4 +1,4 @@ mod steps; mod world; -pub use world::TestingFrameworkWorld; +pub use world::{DeployerKind, TestingFrameworkWorld}; diff --git a/testing-framework/cucumber/src/steps/scenario.rs b/testing-framework/cucumber/src/steps/scenario.rs index 1adccef..227f4e3 100644 --- a/testing-framework/cucumber/src/steps/scenario.rs +++ b/testing-framework/cucumber/src/steps/scenario.rs @@ -1,12 +1,26 @@ use cucumber::given; -use crate::world::{NetworkKind, StepResult, TestingFrameworkWorld, parse_deployer}; +use crate::world::{NetworkKind, StepError, StepResult, TestingFrameworkWorld, parse_deployer}; #[given(expr = "deployer is {string}")] async fn deployer_is(world: &mut TestingFrameworkWorld, deployer: String) -> StepResult { world.set_deployer(parse_deployer(&deployer)?) } +#[given(expr = "we have a CLI deployer specified")] +async fn auto_deployer(world: &mut TestingFrameworkWorld) -> StepResult { + let _unused = world + .deployer + .ok_or(StepError::MissingDeployer) + .inspect_err(|e| { + println!( + "CLI deployer mode not specified, use '--deployer=compose' or '--deployer=local': {}", + e + ) + })?; + Ok(()) +} + #[given(expr = "topology has {int} validators and {int} executors")] async fn topology_has( world: &mut TestingFrameworkWorld,