status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Makes timings not static and ties them to test lifecycle

This commit is contained in:
benbierens 2023-05-04 08:55:20 +02:00
parent 2ed6993b58
commit 5a4a5795b2
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
10 changed files with 56 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
DistTestCore/Codex/CodexAccess.cs
View File

@ -6,10 +6,12 @@ namespace DistTestCore.Codex
public class CodexAccess
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
public CodexAccess(BaseLog log, RunningContainer runningContainer)
public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
{
this.log = log;
this.timeSet = timeSet;
Container = runningContainer;
}
@ -44,7 +46,7 @@ namespace DistTestCore.Codex
{
var ip = Container.Pod.Cluster.IP;
var port = Container.ServicePorts[0].Number;
return new Http(log, ip, port, baseUrl: "/api/codex/v1");
return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)

2
DistTestCore/CodexNodeGroup.cs
View File

@ -64,7 +64,7 @@ namespace DistTestCore
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{
var access = new CodexAccess(lifecycle.Log, c);
var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
EnsureOnline(access);
return factory.CreateOnlineCodexNode(access, this);
}

6
DistTestCore/Configuration.cs
View File

@ -4,13 +4,13 @@ namespace DistTestCore
{
public class Configuration
{
public KubernetesWorkflow.Configuration GetK8sConfiguration()
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{
return new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: "ct-",
kubeConfigFile: null,
operationTimeout: Timing.K8sOperationTimeout(),
retryDelay: Timing.K8sServiceDelay(),
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay(),
locationMap: new[]
{
new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),

44
DistTestCore/DistTest.cs
View File

@ -32,13 +32,11 @@ namespace DistTestCore
{
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
Timing.UseLongTimeouts = false;
try
{
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration());
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
wc.CreateWorkflow().DeleteAllResources();
});
}
@ -58,8 +56,6 @@ namespace DistTestCore
[SetUp]
public void SetUpDistTest()
{
Timing.UseLongTimeouts = ShouldUseLongTimeouts();
if (GlobalTestFailure.HasFailed)
{
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
@ -145,21 +141,6 @@ namespace DistTestCore
}
}
private bool ShouldUseLongTimeouts()
{
// Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
// But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
// if the attribute is present.
var currentTest = TestContext.CurrentContext.Test;
var className = currentTest.ClassName;
var methodName = currentTest.MethodName;
var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
}
private void CreateNewTestLifecycle()
{
var testName = GetCurrentTestName();
@ -167,7 +148,7 @@ namespace DistTestCore
{
lock (lifecycleLock)
{
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration));
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
}
});
}
@ -185,6 +166,27 @@ namespace DistTestCore
});
}
private ITimeSet GetTimeSet()
{
if (ShouldUseLongTimeouts()) return new LongTimeSet();
return new DefaultTimeSet();
}
private bool ShouldUseLongTimeouts()
{
// Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
// But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
// if the attribute is present.
var currentTest = TestContext.CurrentContext.Test;
var className = currentTest.ClassName;
var methodName = currentTest.MethodName;
var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
}
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
{
var result = TestContext.CurrentContext.Result;

14
DistTestCore/Http.cs
View File

@ -10,13 +10,15 @@ namespace DistTestCore
public class Http
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
private readonly string ip;
private readonly int port;
private readonly string baseUrl;
public Http(BaseLog log, string ip, int port, string baseUrl)
public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
{
this.log = log;
this.timeSet = timeSet;
this.ip = ip;
this.port = port;
this.baseUrl = baseUrl;
@ -103,7 +105,7 @@ namespace DistTestCore
log.Debug($"({url}) = '{message}'", 3);
}
private static T Retry<T>(Func<T> operation)
private T Retry<T>(Func<T> operation)
{
var retryCounter = 0;
@ -115,9 +117,9 @@ namespace DistTestCore
}
catch (Exception exception)
{
Timing.HttpCallRetryDelay();
timeSet.HttpCallRetryDelay();
retryCounter++;
if (retryCounter > Timing.HttpCallRetryCount())
if (retryCounter > timeSet.HttpCallRetryCount())
{
Assert.Fail(exception.ToString());
throw;
@ -140,10 +142,10 @@ namespace DistTestCore
}
}
private static HttpClient GetClient()
private HttpClient GetClient()
{
var client = new HttpClient();
client.Timeout = Timing.HttpCallTimeout();
client.Timeout = timeSet.HttpCallTimeout();
return client;
}
}

6
DistTestCore/Metrics/MetricsAccess.cs
View File

@ -14,12 +14,14 @@ namespace DistTestCore.Metrics
public class MetricsAccess : IMetricsAccess
{
private readonly TestLog log;
private readonly ITimeSet timeSet;
private readonly MetricsQuery query;
private readonly RunningContainer node;
public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node)
public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
{
this.log = log;
this.timeSet = timeSet;
this.query = query;
this.node = node;
}
@ -47,7 +49,7 @@ namespace DistTestCore.Metrics
{
var mostRecent = GetMostRecent(metricName);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
{
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
throw new TimeoutException();

4
DistTestCore/Metrics/MetricsAccessFactory.cs
View File

@ -28,8 +28,8 @@ namespace DistTestCore.Metrics
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{
var query = new MetricsQuery(lifecycle.Log, prometheusContainer);
return new MetricsAccess(lifecycle.Log, query, codexContainer);
var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
}
}
}

3
DistTestCore/Metrics/MetricsQuery.cs
View File

@ -9,12 +9,13 @@ namespace DistTestCore.Metrics
{
private readonly Http http;
public MetricsQuery(BaseLog log, RunningContainers runningContainers)
public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
{
RunningContainers = runningContainers;
http = new Http(
log,
timeSet,
runningContainers.RunningPod.Cluster.IP,
runningContainers.Containers[0].ServicePorts[0].Number,
"api/v1");

6
DistTestCore/TestLifecycle.cs
View File

@ -10,10 +10,11 @@ namespace DistTestCore
private readonly WorkflowCreator workflowCreator;
private DateTime testStart = DateTime.MinValue;
public TestLifecycle(TestLog log, Configuration configuration)
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
{
Log = log;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration());
TimeSet = timeSet;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this, workflowCreator);
@ -23,6 +24,7 @@ namespace DistTestCore
}
public TestLog Log { get; }
public ITimeSet TimeSet { get; }
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }

52
DistTestCore/Timing.cs
View File

@ -8,53 +8,11 @@ namespace DistTestCore
{
}
public static class Timing
{
public static bool UseLongTimeouts { get; set; }
public static TimeSpan HttpCallTimeout()
{
return GetTimes().HttpCallTimeout();
}
public static int HttpCallRetryCount()
{
return GetTimes().HttpCallRetryCount();
}
public static void HttpCallRetryDelay()
{
Time.Sleep(GetTimes().HttpCallRetryDelay());
}
public static TimeSpan K8sServiceDelay()
{
return GetTimes().WaitForK8sServiceDelay();
}
public static TimeSpan K8sOperationTimeout()
{
return GetTimes().K8sOperationTimeout();
}
public static TimeSpan WaitForMetricTimeout()
{
return GetTimes().WaitForMetricTimeout();
}
private static ITimeSet GetTimes()
{
if (UseLongTimeouts) return new LongTimeSet();
return new DefaultTimeSet();
}
}
public interface ITimeSet
{
TimeSpan HttpCallTimeout();
int HttpCallRetryCount();
TimeSpan HttpCallRetryDelay();
void HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout();
@ -72,9 +30,9 @@ namespace DistTestCore
return 5;
}
public TimeSpan HttpCallRetryDelay()
public void HttpCallRetryDelay()
{
return TimeSpan.FromSeconds(3);
Time.Sleep(TimeSpan.FromSeconds(3));
}
public TimeSpan WaitForK8sServiceDelay()
@ -105,9 +63,9 @@ namespace DistTestCore
return 2;
}
public TimeSpan HttpCallRetryDelay()
public void HttpCallRetryDelay()
{
return TimeSpan.FromMinutes(5);
Time.Sleep(TimeSpan.FromMinutes(5));
}
public TimeSpan WaitForK8sServiceDelay()