Recover from poisoned locks in node control

testing-framework/core/src/scenario/runtime/node_clients.rs

@@ -56,7 +56,7 @@ impl NodeClients {
     pub fn validator_clients(&self) -> Vec<ApiClient> {
         self.inner
             .read()
-            .expect("node clients lock poisoned")
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
             .validators
             .clone()
     }
@@ -66,7 +66,7 @@ impl NodeClients {
     pub fn executor_clients(&self) -> Vec<ApiClient> {
         self.inner
             .read()
-            .expect("node clients lock poisoned")
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
             .executors
             .clone()
     }
@@ -97,7 +97,11 @@ impl NodeClients {
 
     /// Iterator over all clients.
     pub fn all_clients(&self) -> Vec<ApiClient> {
-        let guard = self.inner.read().expect("node clients lock poisoned");
+        let guard = self
+            .inner
+            .read()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         guard
             .validators
             .iter()
@@ -109,7 +113,11 @@ impl NodeClients {
     #[must_use]
     /// Choose any random client from validators+executors.
     pub fn any_client(&self) -> Option<ApiClient> {
-        let guard = self.inner.read().expect("node clients lock poisoned");
+        let guard = self
+            .inner
+            .read()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         let validator_count = guard.validators.len();
         let executor_count = guard.executors.len();
         let total = validator_count + executor_count;
@@ -132,17 +140,29 @@ impl NodeClients {
     }
 
     pub fn add_validator(&self, client: ApiClient) {
-        let mut guard = self.inner.write().expect("node clients lock poisoned");
+        let mut guard = self
+            .inner
+            .write()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         guard.validators.push(client);
     }
 
     pub fn add_executor(&self, client: ApiClient) {
-        let mut guard = self.inner.write().expect("node clients lock poisoned");
+        let mut guard = self
+            .inner
+            .write()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         guard.executors.push(client);
     }
 
     pub fn clear(&self) {
-        let mut guard = self.inner.write().expect("node clients lock poisoned");
+        let mut guard = self
+            .inner
+            .write()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         guard.validators.clear();
         guard.executors.clear();
     }

testing-framework/deployers/local/src/node_control/mod.rs

@@ -138,12 +138,20 @@ impl LocalDynamicNodes {
 
     #[must_use]
     pub fn node_client(&self, name: &str) -> Option<ApiClient> {
-        let state = self.state.lock().expect("local dynamic lock poisoned");
+        let state = self
+            .state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         state.clients_by_name.get(name).cloned()
     }
 
     pub fn stop_all(&self) {
-        let mut state = self.state.lock().expect("local dynamic lock poisoned");
+        let mut state = self
+            .state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         state.validators.clear();
         state.executors.clear();
         state.peer_ports.clone_from(&self.seed.peer_ports);
@@ -173,7 +181,10 @@ impl LocalDynamicNodes {
     }
 
     pub(crate) fn readiness_nodes(&self) -> Vec<ReadinessNode> {
-        let state = self.state.lock().expect("local dynamic lock poisoned");
+        let state = self
+            .state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
 
         let listen_ports = state
             .validators
@@ -245,7 +256,11 @@ impl LocalDynamicNodes {
         options: StartNodeOptions,
     ) -> Result<StartedNode, LocalDynamicError> {
         let (peer_ports, peer_ports_by_name, node_name, index) = {
-            let state = self.state.lock().expect("local dynamic lock poisoned");
+            let state = self
+                .state
+                .lock()
+                .unwrap_or_else(|poisoned| poisoned.into_inner());
+
             let (index, role_label) = match role {
                 NodeRole::Validator => (state.validator_count, "validator"),
                 NodeRole::Executor => (state.executor_count, "executor"),
@@ -315,7 +330,11 @@ impl LocalDynamicNodes {
 
         self.node_clients.add_validator(client.clone());
 
-        let mut state = self.state.lock().expect("local dynamic lock poisoned");
+        let mut state = self
+            .state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         state.register_validator(node_name, network_port, client.clone(), node);
 
         Ok(client)
@@ -334,7 +353,11 @@ impl LocalDynamicNodes {
 
         self.node_clients.add_executor(client.clone());
 
-        let mut state = self.state.lock().expect("local dynamic lock poisoned");
+        let mut state = self
+            .state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner());
+
         state.register_executor(node_name, network_port, client.clone(), node);
 
         Ok(client)