Changes time.retry to fixed timelength instead of fixed number of retries

Framework/Core/Http.cs

@@ -58,7 +58,7 @@ namespace Core
         {
             lock (httpLock)
             {
-                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
+                return Time.Retry(operation, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
             }
         }
 

Framework/Core/TimeSet.cs

@@ -2,10 +2,31 @@
 {
     public interface ITimeSet
     {
+        /// <summary>
+        /// Timeout for a single HTTP call.
+        /// </summary>
         TimeSpan HttpCallTimeout();
-        int HttpMaxNumberOfRetries();
+
+        /// <summary>
+        /// Maximum total time to attempt to make a successful HTTP call to a service.
+        /// When HTTP calls time out during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan HttpRetryTimeout();
+
+        /// <summary>
+        /// After a failed HTTP call, wait this long before trying again.
+        /// </summary>
         TimeSpan HttpCallRetryDelay();
+
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
         TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
         TimeSpan K8sOperationTimeout();
     }
 
@@ -16,9 +37,9 @@
             return TimeSpan.FromMinutes(3);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 3;
+            return TimeSpan.FromMinutes(10);
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -41,17 +62,17 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromHours(2);
+            return TimeSpan.FromMinutes(30);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 1;
+            return TimeSpan.FromHours(2.2);
         }
 
         public TimeSpan HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(2);
+            return TimeSpan.FromSeconds(20);
         }
 
         public TimeSpan K8sOperationRetryDelay()

Framework/KubernetesWorkflow/K8sController.cs

@@ -712,7 +712,7 @@ namespace KubernetesWorkflow
         {
             return Time.Retry(() => GetPodForDeplomentInternal(deployment),
                 // We will wait up to 1 minute, k8s might be moving pods around.
-                maxRetries: 6,
+                maxTimeout: TimeSpan.FromMinutes(1),
                 retryTime: TimeSpan.FromSeconds(10),
                 description: "Find pod by label for deployment.");
         }

Framework/Utils/Time.cs

@@ -83,35 +83,36 @@
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, 1, description);
+            Retry(action, TimeSpan.FromSeconds(30), description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, 1, description);
+            return Retry(action, TimeSpan.FromSeconds(30), description);
         }
 
-        public static void Retry(Action action, int maxRetries, string description)
+        public static void Retry(Action action, TimeSpan maxTimeout, string description)
         {
-            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+            Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
         }
 
-        public static T Retry<T>(Func<T> action, int maxRetries, string description)
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, string description)
         {
-            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
+            return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
         }
 
-        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
+        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
-            var retries = 0;
+            var tries = 1;
             var exceptions = new List<Exception>();
+
             while (true)
             {
-                if (retries > maxRetries)
+                var duration = DateTime.UtcNow - start;
+                if (duration > maxTimeout)
                 {
-                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
-                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -122,24 +123,25 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
-                    retries++;
+                    tries++;
                 }
 
                 Sleep(retryTime);
             }
         }
 
-        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
-            var retries = 0;
+            var tries = 1;
             var exceptions = new List<Exception>();
+
             while (true)
             {
-                if (retries > maxRetries)
+                var duration = DateTime.UtcNow - start;
+                if (duration > maxTimeout)
                 {
-                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
-                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -149,7 +151,7 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
-                    retries++;
+                    tries++;
                 }
 
                 Sleep(retryTime);

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -95,7 +95,7 @@ namespace CodexTests.BasicTests
                 Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}");
 
                 if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception();
-            }, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events");
+            }, purchase.Expiry + TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5), "Checking SlotFilled events");
         }
 
         private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)

Tools/CodexNetDeployer/Deployer.cs

@@ -255,9 +255,9 @@ namespace CodexNetDeployer
             return TimeSpan.FromSeconds(2);
         }
 
-        public int HttpMaxNumberOfRetries()
+        public TimeSpan HttpRetryTimeout()
         {
-            return 2;
+            return TimeSpan.FromSeconds(30);
         }
 
         public TimeSpan HttpCallTimeout()