diff --git a/Framework/Core/Http.cs b/Framework/Core/Http.cs index 0bd3117..49b3b09 100644 --- a/Framework/Core/Http.cs +++ b/Framework/Core/Http.cs @@ -58,7 +58,7 @@ namespace Core { lock (httpLock) { - return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description); + return Time.Retry(operation, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), description); } } diff --git a/Framework/Core/TimeSet.cs b/Framework/Core/TimeSet.cs index 0b8ba94..0e29c31 100644 --- a/Framework/Core/TimeSet.cs +++ b/Framework/Core/TimeSet.cs @@ -2,10 +2,31 @@ { public interface ITimeSet { + /// + /// Timeout for a single HTTP call. + /// TimeSpan HttpCallTimeout(); - int HttpMaxNumberOfRetries(); + + /// + /// Maximum total time to attempt to make a successful HTTP call to a service. + /// When HTTP calls time out during this timespan, retries will be made. + /// + TimeSpan HttpRetryTimeout(); + + /// + /// After a failed HTTP call, wait this long before trying again. + /// TimeSpan HttpCallRetryDelay(); + + /// + /// After a failed K8s operation, wait this long before trying again. + /// TimeSpan K8sOperationRetryDelay(); + + /// + /// Maximum total time to attempt to perform a successful k8s operation. + /// If k8s operations fail during this timespan, retries will be made. + /// TimeSpan K8sOperationTimeout(); } @@ -16,9 +37,9 @@ return TimeSpan.FromMinutes(3); } - public int HttpMaxNumberOfRetries() + public TimeSpan HttpRetryTimeout() { - return 3; + return TimeSpan.FromMinutes(10); } public TimeSpan HttpCallRetryDelay() @@ -41,17 +62,17 @@ { public TimeSpan HttpCallTimeout() { - return TimeSpan.FromHours(2); + return TimeSpan.FromMinutes(30); } - public int HttpMaxNumberOfRetries() + public TimeSpan HttpRetryTimeout() { - return 1; + return TimeSpan.FromHours(2.2); } public TimeSpan HttpCallRetryDelay() { - return TimeSpan.FromSeconds(2); + return TimeSpan.FromSeconds(20); } public TimeSpan K8sOperationRetryDelay() diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 71430a7..0f50d87 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -712,7 +712,7 @@ namespace KubernetesWorkflow { return Time.Retry(() => GetPodForDeplomentInternal(deployment), // We will wait up to 1 minute, k8s might be moving pods around. - maxRetries: 6, + maxTimeout: TimeSpan.FromMinutes(1), retryTime: TimeSpan.FromSeconds(10), description: "Find pod by label for deployment."); } diff --git a/Framework/Utils/Time.cs b/Framework/Utils/Time.cs index e54ed18..f0f7604 100644 --- a/Framework/Utils/Time.cs +++ b/Framework/Utils/Time.cs @@ -83,35 +83,36 @@ public static void Retry(Action action, string description) { - Retry(action, 1, description); + Retry(action, TimeSpan.FromSeconds(30), description); } public static T Retry(Func action, string description) { - return Retry(action, 1, description); + return Retry(action, TimeSpan.FromSeconds(30), description); } - public static void Retry(Action action, int maxRetries, string description) + public static void Retry(Action action, TimeSpan maxTimeout, string description) { - Retry(action, maxRetries, TimeSpan.FromSeconds(5), description); + Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description); } - public static T Retry(Func action, int maxRetries, string description) + public static T Retry(Func action, TimeSpan maxTimeout, string description) { - return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description); + return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description); } - public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description) + public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description) { var start = DateTime.UtcNow; - var retries = 0; + var tries = 1; var exceptions = new List(); + while (true) { - if (retries > maxRetries) + var duration = DateTime.UtcNow - start; + if (duration > maxTimeout) { - var duration = DateTime.UtcNow - start; - throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions)); + throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions)); } try @@ -122,24 +123,25 @@ catch (Exception ex) { exceptions.Add(ex); - retries++; + tries++; } Sleep(retryTime); } } - public static T Retry(Func action, int maxRetries, TimeSpan retryTime, string description) + public static T Retry(Func action, TimeSpan maxTimeout, TimeSpan retryTime, string description) { var start = DateTime.UtcNow; - var retries = 0; + var tries = 1; var exceptions = new List(); + while (true) { - if (retries > maxRetries) + var duration = DateTime.UtcNow - start; + if (duration > maxTimeout) { - var duration = DateTime.UtcNow - start; - throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions)); + throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions)); } try @@ -149,7 +151,7 @@ catch (Exception ex) { exceptions.Add(ex); - retries++; + tries++; } Sleep(retryTime); diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index d5befe0..f81d7bd 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -95,7 +95,7 @@ namespace CodexTests.BasicTests Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}"); if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception(); - }, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events"); + }, purchase.Expiry + TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5), "Checking SlotFilled events"); } private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer) diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index adea96d..9537096 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -255,9 +255,9 @@ namespace CodexNetDeployer return TimeSpan.FromSeconds(2); } - public int HttpMaxNumberOfRetries() + public TimeSpan HttpRetryTimeout() { - return 2; + return TimeSpan.FromSeconds(30); } public TimeSpan HttpCallTimeout()