feat: add cucumber auto deployer (#2)

- Added an example that selects the deployer (local or docker compose) based on an environment variable.
- Let cucumber's argument parsing environment select scenarios to run - this allows for a flexible test environment.

Cargo.lock

@@ -1528,6 +1528,7 @@ dependencies = [
  "humantime",
  "inventory",
  "itertools 0.14.0",
+ "junit-report",
  "linked-hash-map",
  "pin-project",
  "ref-cast",
@@ -1742,6 +1743,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "derive-getters"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2c35ab6e03642397cdda1dd58abbc05d418aef8e36297f336d5aba060fe8df"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "derive_arbitrary"
 version = "1.4.2"
@@ -3322,6 +3334,18 @@ dependencies = [
  "thiserror 1.0.69",
 ]
 
+[[package]]
+name = "junit-report"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06c3a3342e6720a82d7d179f380e9841b73a1dd49344e33959fdfe571ce56b55"
+dependencies = [
+ "derive-getters",
+ "quick-xml",
+ "strip-ansi-escapes",
+ "time",
+]
+
 [[package]]
 name = "k256"
 version = "0.13.4"
@@ -5947,6 +5971,15 @@ dependencies = [
  "unsigned-varint 0.8.0",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.31.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -6961,6 +6994,15 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
+[[package]]
+name = "strip-ansi-escapes"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025"
+dependencies = [
+ "vte",
+]
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -8226,6 +8268,15 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "vte"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "walkdir"
 version = "2.5.0"

examples/cucumber/features/auto_deployer_smoke.feature

@@ -0,0 +1,34 @@
+Feature: Testing Framework - Auto Local/Compose Deployer
+
+  Scenario: Run auto deployer smoke scenario (tx + DA + liveness)
+    Given we have a CLI deployer specified
+    And topology has 1 validators and 1 executors
+    And run duration is 60 seconds
+    And wallets total funds is 1000000000 split across 50 users
+    And transactions rate is 1 per block
+    And data availability channel rate is 1 per block and blob rate is 1 per block
+    And expect consensus liveness
+    When run scenario
+    Then scenario should succeed
+
+  # Note: This test may fail on slow computers
+  Scenario: Run auto deployer stress smoke scenario (tx + DA + liveness)
+    Given we have a CLI deployer specified
+    And topology has 3 validators and 3 executors
+    And run duration is 120 seconds
+    And wallets total funds is 1000000000 split across 500 users
+    And transactions rate is 10 per block
+    And data availability channel rate is 1 per block and blob rate is 1 per block
+    And expect consensus liveness
+    When run scenario
+    Then scenario should succeed
+
+  Scenario: Run auto deployer stress smoke scenario no liveness (tx + DA + liveness)
+    Given we have a CLI deployer specified
+    And topology has 3 validators and 3 executors
+    And run duration is 120 seconds
+    And wallets total funds is 1000000000 split across 500 users
+    And transactions rate is 10 per block
+    And data availability channel rate is 1 per block and blob rate is 1 per block
+    When run scenario
+    Then scenario should succeed

examples/cucumber/features/local_smoke.feature

@@ -1,6 +1,6 @@
+@local
 Feature: Testing Framework - Local Runner
 
-  @local
   Scenario: Run a local smoke scenario (tx + DA + liveness)
     Given deployer is "local"
     And topology has 1 validators and 1 executors

examples/src/bin/compose_runner.rs

@@ -1,9 +1,8 @@
 use std::{process, time::Duration};
 
 use anyhow::{Context as _, Result};
-use runner_examples::{
-    ChaosBuilderExt as _, ScenarioBuilderExt as _, defaults::Mode, demo, read_env_any,
-};
+use cucumber_ext::DeployerKind;
+use runner_examples::{ChaosBuilderExt as _, ScenarioBuilderExt as _, demo, read_env_any};
 use testing_framework_core::scenario::{Deployer as _, Runner, ScenarioBuilder};
 use testing_framework_runner_compose::{ComposeDeployer, ComposeRunnerError};
 use tracing::{info, warn};
@@ -24,7 +23,7 @@ const DA_BLOB_RATE: u64 = 1;
 
 #[tokio::main]
 async fn main() {
-    runner_examples::defaults::init_node_log_dir_defaults(Mode::Compose);
+    runner_examples::defaults::init_node_log_dir_defaults(DeployerKind::Compose);
 
     tracing_subscriber::fmt::init();
 

examples/src/bin/cucumber_auto.rs

@@ -0,0 +1,66 @@
+/// Usage: Set the environment variable CUCUMBER_DEPLOYER_COMPOSE to use the
+/// Compose deployer. Otherwise, the Local deployer is used by default.
+///
+/// Example using docker compose deployer:
+/// ```sh
+/// CUCUMBER_DEPLOYER_COMPOSE=1 cargo run -p runner-examples --bin cucumber_auto -- --name "Run auto deployer smoke scenario"
+/// ```
+/// Example using local deployer:
+/// ```sh
+/// cargo run -p runner-examples --bin cucumber_auto --  --name "Run auto deployer smoke scenario"
+/// ```
+use std::{fs, io};
+
+use cucumber::{World, WriterExt, writer, writer::Verbosity};
+use cucumber_ext::{DeployerKind, TestingFrameworkWorld};
+use runner_examples::defaults::{init_logging_defaults, init_node_log_dir_defaults, init_tracing};
+
+#[tokio::main]
+async fn main() {
+    println!("args: {:?}", std::env::args());
+
+    let deployer = if std::env::var("CUCUMBER_DEPLOYER_COMPOSE").ok().is_some() {
+        DeployerKind::Compose
+    } else {
+        DeployerKind::Local
+    };
+    println!("Running with '{:?}'", deployer);
+
+    init_logging_defaults();
+    init_node_log_dir_defaults(deployer);
+    init_tracing();
+
+    // Print current directory for debugging
+    if let Ok(current_dir) = std::env::current_dir() {
+        println!("Current directory: {:?}", current_dir);
+    }
+
+    let file = fs::File::create("cucumber-output-junit.xml").unwrap();
+    let world = TestingFrameworkWorld::cucumber()
+        .repeat_failed()
+        // following config needed to use eprint statements in the tests
+        .max_concurrent_scenarios(1)
+        .fail_on_skipped()
+        .fail_fast()
+        .with_writer(
+            writer::Summarize::new(writer::Basic::new(
+                io::stdout(),
+                writer::Coloring::Auto,
+                Verbosity::ShowWorldAndDocString,
+            ))
+            .tee::<TestingFrameworkWorld, _>(writer::JUnit::for_tee(file, 0))
+            .normalized(),
+        )
+        .before(move |feature, _rule, scenario, world| {
+            Box::pin(async move {
+                println!(
+                    "\nStarting '{}' : '{}' : '{}'\n",
+                    feature.name, scenario.keyword, scenario.name
+                ); // This will be printed into the stdout_buffer
+                if let Err(e) = world.set_deployer(deployer) {
+                    panic!("Failed to set deployer: {}", e);
+                }
+            })
+        });
+    world.run_and_exit("examples/cucumber/features/").await;
+}

examples/src/bin/cucumber_compose.rs

@@ -1,13 +1,11 @@
 use cucumber::World;
-use cucumber_ext::TestingFrameworkWorld;
-use runner_examples::defaults::{
-    Mode, init_logging_defaults, init_node_log_dir_defaults, init_tracing,
-};
+use cucumber_ext::{DeployerKind, TestingFrameworkWorld};
+use runner_examples::defaults::{init_logging_defaults, init_node_log_dir_defaults, init_tracing};
 
 #[tokio::main]
 async fn main() {
     init_logging_defaults();
-    init_node_log_dir_defaults(Mode::Compose);
+    init_node_log_dir_defaults(DeployerKind::Compose);
     init_tracing();
 
     TestingFrameworkWorld::run("examples/cucumber/features/compose_smoke.feature").await;

examples/src/bin/cucumber_host.rs

@@ -1,13 +1,11 @@
 use cucumber::World;
-use cucumber_ext::TestingFrameworkWorld;
-use runner_examples::defaults::{
-    Mode, init_logging_defaults, init_node_log_dir_defaults, init_tracing,
-};
+use cucumber_ext::{DeployerKind, TestingFrameworkWorld};
+use runner_examples::defaults::{init_logging_defaults, init_node_log_dir_defaults, init_tracing};
 
 #[tokio::main]
 async fn main() {
     init_logging_defaults();
-    init_node_log_dir_defaults(Mode::Host);
+    init_node_log_dir_defaults(DeployerKind::Local);
     init_tracing();
 
     TestingFrameworkWorld::run("examples/cucumber/features/local_smoke.feature").await;

examples/src/bin/local_runner.rs

@@ -1,7 +1,8 @@
 use std::{env, process, time::Duration};
 
 use anyhow::{Context as _, Result};
-use runner_examples::{ScenarioBuilderExt as _, defaults::Mode, demo, read_env_any};
+use cucumber_ext::DeployerKind;
+use runner_examples::{ScenarioBuilderExt as _, demo, read_env_any};
 use testing_framework_core::scenario::{Deployer as _, Runner, ScenarioBuilder};
 use testing_framework_runner_local::LocalDeployer;
 use tracing::{info, warn};
@@ -14,7 +15,7 @@ const SMOKE_RUN_SECS_MAX: u64 = 30;
 
 #[tokio::main]
 async fn main() {
-    runner_examples::defaults::init_node_log_dir_defaults(Mode::Host);
+    runner_examples::defaults::init_node_log_dir_defaults(DeployerKind::Local);
 
     tracing_subscriber::fmt::init();
 

examples/src/cucumber.rs

@@ -1,27 +0,0 @@
-use cucumber::World;
-use cucumber_ext::TestingFrameworkWorld;
-
-pub use crate::defaults::Mode;
-
-const FEATURES_PATH: &str = "examples/cucumber/features";
-
-pub use crate::defaults::{init_logging_defaults, init_node_log_dir_defaults, init_tracing};
-
-fn is_compose(
-    feature: &cucumber::gherkin::Feature,
-    scenario: &cucumber::gherkin::Scenario,
-) -> bool {
-    scenario.tags.iter().any(|tag| tag == "compose")
-        || feature.tags.iter().any(|tag| tag == "compose")
-}
-
-pub async fn run(mode: Mode) {
-    TestingFrameworkWorld::cucumber()
-        .with_default_cli()
-        .max_concurrent_scenarios(Some(1))
-        .filter_run(FEATURES_PATH, move |feature, _, scenario| match mode {
-            Mode::Host => !is_compose(feature, scenario),
-            Mode::Compose => is_compose(feature, scenario),
-        })
-        .await;
-}

examples/src/defaults.rs

@@ -3,14 +3,9 @@ use std::{
     path::{Path, PathBuf},
 };
 
+use cucumber_ext::DeployerKind;
 use tracing_subscriber::{EnvFilter, fmt};
 
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
-pub enum Mode {
-    Host,
-    Compose,
-}
-
 const DEFAULT_NODE_LOG_DIR_REL: &str = ".tmp/node-logs";
 const DEFAULT_CONTAINER_NODE_LOG_DIR: &str = "/tmp/node-logs";
 
@@ -31,7 +26,7 @@ pub fn init_logging_defaults() {
     set_default_env("RUST_LOG", "info");
 }
 
-pub fn init_node_log_dir_defaults(mode: Mode) {
+pub fn init_node_log_dir_defaults(deployer: DeployerKind) {
     if env::var_os("NOMOS_LOG_DIR").is_some() {
         return;
     }
@@ -39,9 +34,9 @@ pub fn init_node_log_dir_defaults(mode: Mode) {
     let host_dir = repo_root().join(DEFAULT_NODE_LOG_DIR_REL);
     let _ = fs::create_dir_all(&host_dir);
 
-    match mode {
-        Mode::Host => set_default_env("NOMOS_LOG_DIR", &host_dir.display().to_string()),
-        Mode::Compose => set_default_env("NOMOS_LOG_DIR", DEFAULT_CONTAINER_NODE_LOG_DIR),
+    match deployer {
+        DeployerKind::Local => set_default_env("NOMOS_LOG_DIR", &host_dir.display().to_string()),
+        DeployerKind::Compose => set_default_env("NOMOS_LOG_DIR", DEFAULT_CONTAINER_NODE_LOG_DIR),
     }
 }
 

examples/src/lib.rs

@@ -1,4 +1,3 @@
-pub mod cucumber;
 pub mod defaults;
 pub mod demo;
 pub mod env;

testing-framework/cucumber/Cargo.toml

@@ -10,7 +10,7 @@ repository.workspace  = true
 version.workspace     = true
 
 [dependencies]
-cucumber                         = { version = "0.22.0", features = ["default", "macros"] }
+cucumber                         = { version = "0.22.0", features = ["default", "macros", "output-junit"] }
 testing-framework-core           = { workspace = true }
 testing-framework-runner-compose = { workspace = true }
 testing-framework-runner-local   = { workspace = true }

testing-framework/cucumber/src/lib.rs

@@ -1,4 +1,4 @@
 mod steps;
 mod world;
 
-pub use world::TestingFrameworkWorld;
+pub use world::{DeployerKind, TestingFrameworkWorld};

testing-framework/cucumber/src/steps/scenario.rs

@@ -1,12 +1,26 @@
 use cucumber::given;
 
-use crate::world::{NetworkKind, StepResult, TestingFrameworkWorld, parse_deployer};
+use crate::world::{NetworkKind, StepError, StepResult, TestingFrameworkWorld, parse_deployer};
 
 #[given(expr = "deployer is {string}")]
 async fn deployer_is(world: &mut TestingFrameworkWorld, deployer: String) -> StepResult {
     world.set_deployer(parse_deployer(&deployer)?)
 }
 
+#[given(expr = "we have a CLI deployer specified")]
+async fn auto_deployer(world: &mut TestingFrameworkWorld) -> StepResult {
+    let _unused = world
+        .deployer
+        .ok_or(StepError::MissingDeployer)
+        .inspect_err(|e| {
+            println!(
+                "CLI deployer mode not specified, use '--deployer=compose' or '--deployer=local': {}",
+                e
+            )
+        })?;
+    Ok(())
+}
+
 #[given(expr = "topology has {int} validators and {int} executors")]
 async fn topology_has(
     world: &mut TestingFrameworkWorld,