Changes time.retry to fixed timelength instead of fixed number of retries

This commit is contained in:
benbierens 2024-05-02 08:41:20 +02:00
parent dd36929a81
commit e187bfc941
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
6 changed files with 53 additions and 30 deletions

View File

@ -58,7 +58,7 @@ namespace Core
{
lock (httpLock)
{
return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
return Time.Retry(operation, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
}
}

View File

@ -2,10 +2,31 @@
{
public interface ITimeSet
{
/// <summary>
/// Timeout for a single HTTP call.
/// </summary>
TimeSpan HttpCallTimeout();
int HttpMaxNumberOfRetries();
/// <summary>
/// Maximum total time to attempt to make a successful HTTP call to a service.
/// When HTTP calls time out during this timespan, retries will be made.
/// </summary>
TimeSpan HttpRetryTimeout();
/// <summary>
/// After a failed HTTP call, wait this long before trying again.
/// </summary>
TimeSpan HttpCallRetryDelay();
/// <summary>
/// After a failed K8s operation, wait this long before trying again.
/// </summary>
TimeSpan K8sOperationRetryDelay();
/// <summary>
/// Maximum total time to attempt to perform a successful k8s operation.
/// If k8s operations fail during this timespan, retries will be made.
/// </summary>
TimeSpan K8sOperationTimeout();
}
@ -16,9 +37,9 @@
return TimeSpan.FromMinutes(3);
}
public int HttpMaxNumberOfRetries()
public TimeSpan HttpRetryTimeout()
{
return 3;
return TimeSpan.FromMinutes(10);
}
public TimeSpan HttpCallRetryDelay()
@ -41,17 +62,17 @@
{
public TimeSpan HttpCallTimeout()
{
return TimeSpan.FromHours(2);
return TimeSpan.FromMinutes(30);
}
public int HttpMaxNumberOfRetries()
public TimeSpan HttpRetryTimeout()
{
return 1;
return TimeSpan.FromHours(2.2);
}
public TimeSpan HttpCallRetryDelay()
{
return TimeSpan.FromSeconds(2);
return TimeSpan.FromSeconds(20);
}
public TimeSpan K8sOperationRetryDelay()

View File

@ -712,7 +712,7 @@ namespace KubernetesWorkflow
{
return Time.Retry(() => GetPodForDeplomentInternal(deployment),
// We will wait up to 1 minute, k8s might be moving pods around.
maxRetries: 6,
maxTimeout: TimeSpan.FromMinutes(1),
retryTime: TimeSpan.FromSeconds(10),
description: "Find pod by label for deployment.");
}

View File

@ -83,35 +83,36 @@
public static void Retry(Action action, string description)
{
Retry(action, 1, description);
Retry(action, TimeSpan.FromSeconds(30), description);
}
public static T Retry<T>(Func<T> action, string description)
{
return Retry(action, 1, description);
return Retry(action, TimeSpan.FromSeconds(30), description);
}
public static void Retry(Action action, int maxRetries, string description)
public static void Retry(Action action, TimeSpan maxTimeout, string description)
{
Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
}
public static T Retry<T>(Func<T> action, int maxRetries, string description)
public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, string description)
{
return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
return Retry(action, maxTimeout, TimeSpan.FromSeconds(5), description);
}
public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;
var retries = 0;
var tries = 1;
var exceptions = new List<Exception>();
while (true)
{
if (retries > maxRetries)
var duration = DateTime.UtcNow - start;
if (duration > maxTimeout)
{
var duration = DateTime.UtcNow - start;
throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
}
try
@ -122,24 +123,25 @@
catch (Exception ex)
{
exceptions.Add(ex);
retries++;
tries++;
}
Sleep(retryTime);
}
}
public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;
var retries = 0;
var tries = 1;
var exceptions = new List<Exception>();
while (true)
{
if (retries > maxRetries)
var duration = DateTime.UtcNow - start;
if (duration > maxTimeout)
{
var duration = DateTime.UtcNow - start;
throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
}
try
@ -149,7 +151,7 @@
catch (Exception ex)
{
exceptions.Add(ex);
retries++;
tries++;
}
Sleep(retryTime);

View File

@ -95,7 +95,7 @@ namespace CodexTests.BasicTests
Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}");
if (slotFilledEvents.Length != purchase.MinRequiredNumberOfNodes) throw new Exception();
}, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events");
}, purchase.Expiry + TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5), "Checking SlotFilled events");
}
private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer)

View File

@ -255,9 +255,9 @@ namespace CodexNetDeployer
return TimeSpan.FromSeconds(2);
}
public int HttpMaxNumberOfRetries()
public TimeSpan HttpRetryTimeout()
{
return 2;
return TimeSpan.FromSeconds(30);
}
public TimeSpan HttpCallTimeout()