diff --git a/Framework/Core/Http.cs b/Framework/Core/Http.cs
index 0bd3117..49b3b09 100644
--- a/Framework/Core/Http.cs
+++ b/Framework/Core/Http.cs
@@ -58,7 +58,7 @@ namespace Core
{
lock (httpLock)
{
- return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
+ return Time.Retry(operation, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
}
}
diff --git a/Framework/Core/TimeSet.cs b/Framework/Core/TimeSet.cs
index 0b8ba94..0e29c31 100644
--- a/Framework/Core/TimeSet.cs
+++ b/Framework/Core/TimeSet.cs
@@ -2,10 +2,31 @@
{
public interface ITimeSet
{
+ ///
+ /// Timeout for a single HTTP call.
+ ///
TimeSpan HttpCallTimeout();
- int HttpMaxNumberOfRetries();
+
+ ///
+ /// Maximum total time to attempt to make a successful HTTP call to a service.
+ /// When HTTP calls time out during this timespan, retries will be made.
+ ///
+ TimeSpan HttpRetryTimeout();
+
+ ///
+ /// After a failed HTTP call, wait this long before trying again.
+ ///
TimeSpan HttpCallRetryDelay();
+
+ ///
+ /// After a failed K8s operation, wait this long before trying again.
+ ///
TimeSpan K8sOperationRetryDelay();
+
+ ///
+ /// Maximum total time to attempt to perform a successful k8s operation.
+ /// If k8s operations fail during this timespan, retries will be made.
+ ///
TimeSpan K8sOperationTimeout();
}
@@ -16,9 +37,9 @@
return TimeSpan.FromMinutes(3);
}
- public int HttpMaxNumberOfRetries()
+ public TimeSpan HttpRetryTimeout()
{
- return 3;
+ return TimeSpan.FromMinutes(10);
}
public TimeSpan HttpCallRetryDelay()
@@ -41,17 +62,17 @@
{
public TimeSpan HttpCallTimeout()
{
- return TimeSpan.FromHours(2);
+ return TimeSpan.FromMinutes(30);
}
- public int HttpMaxNumberOfRetries()
+ public TimeSpan HttpRetryTimeout()
{
- return 1;
+ return TimeSpan.FromHours(2.2);
}
public TimeSpan HttpCallRetryDelay()
{
- return TimeSpan.FromSeconds(2);
+ return TimeSpan.FromSeconds(20);
}
public TimeSpan K8sOperationRetryDelay()
diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs
index 71430a7..0f50d87 100644
--- a/Framework/KubernetesWorkflow/K8sController.cs
+++ b/Framework/KubernetesWorkflow/K8sController.cs
@@ -712,7 +712,7 @@ namespace KubernetesWorkflow
{
return Time.Retry(() => GetPodForDeplomentInternal(deployment),
// We will wait up to 1 minute, k8s might be moving pods around.
- maxRetries: 6,
+ maxTimeout: TimeSpan.FromMinutes(1),
retryTime: TimeSpan.FromSeconds(10),
description: "Find pod by label for deployment.");
}
diff --git a/Framework/Utils/Time.cs b/Framework/Utils/Time.cs
index e54ed18..f0f7604 100644
--- a/Framework/Utils/Time.cs
+++ b/Framework/Utils/Time.cs
@@ -83,35 +83,36 @@
public static void Retry(Action action, string description)
{
- Retry(action, 1, description);
+ Retry(action, TimeSpan.FromSeconds(30), description);
}
public static T Retry(Func action, string description)
{
- return Retry(action, 1, description);
+ return Retry(action, TimeSpan.FromSeconds(30), description);
}
- public static void Retry(Action action, int maxRetries, string description)
+ public static void Retry(Action action, TimeSpan maxTimeout, string description)
{
- Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+ Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
}
- public static T Retry(Func action, int maxRetries, string description)
+ public static T Retry(Func action, TimeSpan maxTimeout, string description)
{
- return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+ return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
}
- public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
+ public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;
- var retries = 0;
+ var tries = 1;
var exceptions = new List();
+
while (true)
{
- if (retries > maxRetries)
+ var duration = DateTime.UtcNow - start;
+ if (duration > maxTimeout)
{
- var duration = DateTime.UtcNow - start;
- throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
+ throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
}
try
@@ -122,24 +123,25 @@
catch (Exception ex)
{
exceptions.Add(ex);
- retries++;
+ tries++;
}
Sleep(retryTime);
}
}
- public static T Retry(Func action, int maxRetries, TimeSpan retryTime, string description)
+ public static T Retry(Func action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;
- var retries = 0;
+ var tries = 1;
var exceptions = new List();
+
while (true)
{
- if (retries > maxRetries)
+ var duration = DateTime.UtcNow - start;
+ if (duration > maxTimeout)
{
- var duration = DateTime.UtcNow - start;
- throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
+ throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
}
try
@@ -149,7 +151,7 @@
catch (Exception ex)
{
exceptions.Add(ex);
- retries++;
+ tries++;
}
Sleep(retryTime);
diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs
index d5befe0..f81d7bd 100644
--- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs
+++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs
@@ -95,7 +95,7 @@ namespace CodexTests.BasicTests
Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}");
if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception();
- }, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events");
+ }, purchase.Expiry + TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5), "Checking SlotFilled events");
}
private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)
diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs
index adea96d..9537096 100644
--- a/Tools/CodexNetDeployer/Deployer.cs
+++ b/Tools/CodexNetDeployer/Deployer.cs
@@ -255,9 +255,9 @@ namespace CodexNetDeployer
return TimeSpan.FromSeconds(2);
}
- public int HttpMaxNumberOfRetries()
+ public TimeSpan HttpRetryTimeout()
{
- return 2;
+ return TimeSpan.FromSeconds(30);
}
public TimeSpan HttpCallTimeout()