workflows: retry tx submission

Also bump pinned NOMOS_NODE_REV.

testing-framework/workflows/src/workloads/util.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
 
 use nomos_core::{
     block::Block,
@@ -11,6 +11,9 @@ use rand::{seq::SliceRandom as _, thread_rng};
 use testing_framework_core::scenario::{DynError, RunContext};
 use tracing::debug;
 
+const SUBMIT_RETRIES: usize = 5;
+const SUBMIT_RETRY_DELAY: Duration = Duration::from_millis(500);
+
 /// Scans a block and invokes the matcher for every operation until it returns
 /// `Some(...)`. Returns `None` when no matching operation is found.
 pub fn find_channel_op<F>(block: &Block<SignedMantleTx>, matcher: &mut F) -> Option<MsgId>
@@ -51,22 +54,29 @@ pub async fn submit_transaction_via_cluster(
     executor_clients.shuffle(&mut thread_rng());
 
     let clients = validator_clients.into_iter().chain(executor_clients);
+    let mut clients: Vec<_> = clients.collect();
     let mut last_err = None;
 
-    for client in clients {
-        let url = client.base_url().clone();
-        debug!(?tx_hash, %url, "submitting transaction to client");
-        match client
-            .submit_transaction(&tx)
-            .await
-            .map_err(|err| -> DynError { err.into() })
-        {
-            Ok(()) => return Ok(()),
-            Err(err) => {
-                debug!(?tx_hash, %url, "transaction submission failed");
-                last_err = Some(err);
+    for attempt in 0..SUBMIT_RETRIES {
+        clients.shuffle(&mut thread_rng());
+
+        for client in &clients {
+            let url = client.base_url().clone();
+            debug!(?tx_hash, %url, attempt, "submitting transaction to client");
+            match client
+                .submit_transaction(&tx)
+                .await
+                .map_err(|err| -> DynError { err.into() })
+            {
+                Ok(()) => return Ok(()),
+                Err(err) => {
+                    debug!(?tx_hash, %url, attempt, "transaction submission failed");
+                    last_err = Some(err);
+                }
             }
         }
+
+        tokio::time::sleep(SUBMIT_RETRY_DELAY).await;
     }
 
     Err(last_err.unwrap_or_else(|| "cluster client exhausted all nodes".into()))

versions.env

@@ -1,7 +1,7 @@
 VERSION=v0.3.1
 NOMOS_BUNDLE_VERSION=v4
 # Pinned nomos-node revision used for CI builds and binary bundles.
-NOMOS_NODE_REV=ad104981ca79da20183550b5aced9e49773fb6d5
+NOMOS_NODE_REV=6bdb09567d21cd1e53527846a9cd48493ad49387
 
 # Optional: local nomos-node checkout override (do not commit absolute paths).
 # NOMOS_NODE_PATH=