namespace Utils { public class Retry { private readonly string description; private readonly TimeSpan maxTimeout; private readonly TimeSpan sleepAfterFail; private readonly Action onFail; public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action onFail) { this.description = description; this.maxTimeout = maxTimeout; this.sleepAfterFail = sleepAfterFail; this.onFail = onFail; } public void Run(Action task) { var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail); run.Run(); } public T Run(Func task) { T? result = default; var run = new RetryRun(description, () => { result = task(); }, maxTimeout, sleepAfterFail, onFail); run.Run(); return result!; } private class RetryRun { private readonly string description; private readonly Action task; private readonly TimeSpan maxTimeout; private readonly TimeSpan sleepAfterFail; private readonly Action onFail; private readonly DateTime start = DateTime.UtcNow; private readonly List failures = new List(); private int tryNumber; private DateTime tryStart; public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action onFail) { this.description = description; this.task = task; this.maxTimeout = maxTimeout; this.sleepAfterFail = sleepAfterFail; this.onFail = onFail; tryNumber = 0; tryStart = DateTime.UtcNow; } public void Run() { while (true) { CheckMaximums(); tryNumber++; tryStart = DateTime.UtcNow; try { task(); return; } catch (Exception ex) { var failure = CaptureFailure(ex); onFail(failure); Time.Sleep(sleepAfterFail); } } } private Failure CaptureFailure(Exception ex) { var f = new Failure(ex, DateTime.UtcNow - tryStart, tryNumber); failures.Add(f); return f; } private void CheckMaximums() { if (Duration() > maxTimeout) Fail(); } private void Fail() { throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}", new AggregateException(failures.Select(f => f.Exception))); } private string GetFailureReport() { return Environment.NewLine + string.Join(Environment.NewLine, failures.Select(f => f.Describe())); } private TimeSpan Duration() { return DateTime.UtcNow - start; } } } public class Failure { public Failure(Exception exception, TimeSpan duration, int tryNumber) { Exception = exception; Duration = duration; TryNumber = tryNumber; } public Exception Exception { get; } public TimeSpan Duration { get; } public int TryNumber { get; } public string Describe() { return $"Try {TryNumber} failed after {Time.FormatDuration(Duration)} with exception '{Exception}'"; } } }