Makes timings not static and ties them to test lifecycle

DistTestCore/Codex/CodexAccess.cs

@@ -6,10 +6,12 @@ namespace DistTestCore.Codex
     public class CodexAccess
     {
         private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
 
-        public CodexAccess(BaseLog log, RunningContainer runningContainer)
+        public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
         {
             this.log = log;
+            this.timeSet = timeSet;
             Container = runningContainer;
         }
 
@@ -44,7 +46,7 @@ namespace DistTestCore.Codex
         {
             var ip = Container.Pod.Cluster.IP;
             var port = Container.ServicePorts[0].Number;
-            return new Http(log, ip, port, baseUrl: "/api/codex/v1");
+            return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
         }
 
         public string ConnectToPeer(string peerId, string peerMultiAddress)

DistTestCore/CodexNodeGroup.cs

@@ -64,7 +64,7 @@ namespace DistTestCore
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
         {
-            var access = new CodexAccess(lifecycle.Log, c);
+            var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
             EnsureOnline(access);
             return factory.CreateOnlineCodexNode(access, this);
         }

DistTestCore/Configuration.cs

@@ -4,13 +4,13 @@ namespace DistTestCore
 {
     public class Configuration
     {
-        public KubernetesWorkflow.Configuration GetK8sConfiguration()
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
                 k8sNamespacePrefix: "ct-",
                 kubeConfigFile: null,
-                operationTimeout: Timing.K8sOperationTimeout(),
+                operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: Timing.K8sServiceDelay(),
+                retryDelay: timeSet.WaitForK8sServiceDelay(),
                 locationMap: new[]
                 {
                     new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),

DistTestCore/DistTest.cs

@@ -32,13 +32,11 @@ namespace DistTestCore
         {
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
-            Timing.UseLongTimeouts = false;
-
             try
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration());
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -58,8 +56,6 @@ namespace DistTestCore
         [SetUp]
         public void SetUpDistTest()
         {
-            Timing.UseLongTimeouts = ShouldUseLongTimeouts();
-
             if (GlobalTestFailure.HasFailed)
             {
                 Assert.Inconclusive("Skip test: Previous test failed during clean up.");
@@ -145,21 +141,6 @@ namespace DistTestCore
             }
         }
 
-        private bool ShouldUseLongTimeouts()
-        {
-            // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
-            // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
-            // if the attribute is present.
-            var currentTest = TestContext.CurrentContext.Test;
-            var className = currentTest.ClassName;
-            var methodName = currentTest.MethodName;
-
-            var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
-            var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
-
-            return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
-        }
-
         private void CreateNewTestLifecycle()
         {
             var testName = GetCurrentTestName();
@@ -167,7 +148,7 @@ namespace DistTestCore
             {
                 lock (lifecycleLock)
                 {
-                    lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration));
+                    lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
                 }
             });
         }
@@ -185,6 +166,27 @@ namespace DistTestCore
             });
         }
 
+        private ITimeSet GetTimeSet()
+        {
+            if (ShouldUseLongTimeouts()) return new LongTimeSet();
+            return new DefaultTimeSet();
+        }
+
+        private bool ShouldUseLongTimeouts()
+        {
+            // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
+            // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
+            // if the attribute is present.
+            var currentTest = TestContext.CurrentContext.Test;
+            var className = currentTest.ClassName;
+            var methodName = currentTest.MethodName;
+
+            var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
+            var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
+
+            return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
+        }
+
         private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
         {
             var result = TestContext.CurrentContext.Result;

DistTestCore/Http.cs

@@ -10,13 +10,15 @@ namespace DistTestCore
     public class Http
     {
         private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
         private readonly string ip;
         private readonly int port;
         private readonly string baseUrl;
 
-        public Http(BaseLog log, string ip, int port, string baseUrl)
+        public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
         {
             this.log = log;
+            this.timeSet = timeSet;
             this.ip = ip;
             this.port = port;
             this.baseUrl = baseUrl;
@@ -103,7 +105,7 @@ namespace DistTestCore
             log.Debug($"({url}) = '{message}'", 3);
         }
 
-        private static T Retry<T>(Func<T> operation)
+        private T Retry<T>(Func<T> operation)
         {
             var retryCounter = 0;
 
@@ -115,9 +117,9 @@ namespace DistTestCore
                 }
                 catch (Exception exception)
                 {
-                    Timing.HttpCallRetryDelay();
+                    timeSet.HttpCallRetryDelay();
                     retryCounter++;
-                    if (retryCounter > Timing.HttpCallRetryCount())
+                    if (retryCounter > timeSet.HttpCallRetryCount())
                     {
                         Assert.Fail(exception.ToString());
                         throw;
@@ -140,10 +142,10 @@ namespace DistTestCore
             }
         }
 
-        private static HttpClient GetClient()
+        private HttpClient GetClient()
         {
             var client = new HttpClient();
-            client.Timeout = Timing.HttpCallTimeout();
+            client.Timeout = timeSet.HttpCallTimeout();
             return client;
         }
     }

DistTestCore/Metrics/MetricsAccess.cs

@@ -14,12 +14,14 @@ namespace DistTestCore.Metrics
     public class MetricsAccess : IMetricsAccess
     {
         private readonly TestLog log;
+        private readonly ITimeSet timeSet;
         private readonly MetricsQuery query;
         private readonly RunningContainer node;
 
-        public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node)
+        public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
         {
             this.log = log;
+            this.timeSet = timeSet;
             this.query = query;
             this.node = node;
         }
@@ -47,7 +49,7 @@ namespace DistTestCore.Metrics
             {
                 var mostRecent = GetMostRecent(metricName);
                 if (mostRecent != null) return mostRecent;
-                if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
+                if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
                 {
                     Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
                     throw new TimeoutException();

DistTestCore/Metrics/MetricsAccessFactory.cs

@@ -28,8 +28,8 @@ namespace DistTestCore.Metrics
 
         public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
         {
-            var query = new MetricsQuery(lifecycle.Log, prometheusContainer);
+            var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
-            return new MetricsAccess(lifecycle.Log, query, codexContainer);
+            return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
         }
     }
 }

DistTestCore/Metrics/MetricsQuery.cs

@@ -9,12 +9,13 @@ namespace DistTestCore.Metrics
     {
         private readonly Http http;
 
-        public MetricsQuery(BaseLog log, RunningContainers runningContainers)
+        public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
         {
             RunningContainers = runningContainers;
 
             http = new Http(
                 log,
+                timeSet,
                 runningContainers.RunningPod.Cluster.IP,
                 runningContainers.Containers[0].ServicePorts[0].Number,
                 "api/v1");

DistTestCore/TestLifecycle.cs

@@ -10,10 +10,11 @@ namespace DistTestCore
         private readonly WorkflowCreator workflowCreator;
         private DateTime testStart = DateTime.MinValue;
 
-        public TestLifecycle(TestLog log, Configuration configuration)
+        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
         {
             Log = log;
-            workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration());
+            TimeSet = timeSet;
+            workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
 
             FileManager = new FileManager(Log, configuration);
             CodexStarter = new CodexStarter(this, workflowCreator);
@@ -23,6 +24,7 @@ namespace DistTestCore
         }
 
         public TestLog Log { get; }
+        public ITimeSet TimeSet { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }

DistTestCore/Timing.cs

@@ -8,53 +8,11 @@ namespace DistTestCore
     {
     }
 
-    public static class Timing
-    {
-        public static bool UseLongTimeouts { get; set; }
-
-
-        public static TimeSpan HttpCallTimeout()
-        {
-            return GetTimes().HttpCallTimeout();
-        }
-
-        public static int HttpCallRetryCount()
-        {
-            return GetTimes().HttpCallRetryCount();
-        }
-
-        public static void HttpCallRetryDelay()
-        {
-            Time.Sleep(GetTimes().HttpCallRetryDelay());
-        }
-
-        public static TimeSpan K8sServiceDelay()
-        {
-            return GetTimes().WaitForK8sServiceDelay();
-        }
-
-        public static TimeSpan K8sOperationTimeout()
-        {
-            return GetTimes().K8sOperationTimeout();
-        }
-
-        public static TimeSpan WaitForMetricTimeout()
-        {
-            return GetTimes().WaitForMetricTimeout();
-        }
-
-        private static ITimeSet GetTimes()
-        {
-            if (UseLongTimeouts) return new LongTimeSet();
-            return new DefaultTimeSet();
-        }
-    }
-
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
         int HttpCallRetryCount();
-        TimeSpan HttpCallRetryDelay();
+        void HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
         TimeSpan WaitForMetricTimeout();
@@ -72,9 +30,9 @@ namespace DistTestCore
             return 5;
         }
 
-        public TimeSpan HttpCallRetryDelay()
+        public void HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(3);
+            Time.Sleep(TimeSpan.FromSeconds(3));
         }
 
         public TimeSpan WaitForK8sServiceDelay()
@@ -105,9 +63,9 @@ namespace DistTestCore
             return 2;
         }
 
-        public TimeSpan HttpCallRetryDelay()
+        public void HttpCallRetryDelay()
         {
-            return TimeSpan.FromMinutes(5);
+            Time.Sleep(TimeSpan.FromMinutes(5));
         }
 
         public TimeSpan WaitForK8sServiceDelay()