status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove run ID and introduce deploy ID (#93) 

This PR removes the notion of a run id and replaces it with a deploy id in continuous tests. Deploy ids can be set at deploy time only (duh), and will be picked up by the test runner from the deploy file on subsequent runs of the continuous test runner. As a consequence, the deploy id becomes a deployer parameter, and can no longer be overridden at the runner. For non-continuous tests, the deploy ID is created on-the-fly.
This commit is contained in:
Giuliano Mega 2024-02-22 10:41:07 -03:00 committed by GitHub
parent 930c44f037
commit ec0f7a6790
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 108 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ProjectPlugins/CodexPlugin/CodexDeployment.cs
View File

@ -6,8 +6,12 @@ namespace CodexPlugin
{
public class CodexDeployment
{
public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment, CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer, RunningContainers? discordBotContainer, DeploymentMetadata metadata)
public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
RunningContainers? discordBotContainer, DeploymentMetadata metadata,
String id)
{
Id = id;
CodexInstances = codexInstances;
GethDeployment = gethDeployment;
CodexContractsDeployment = codexContractsDeployment;
@ -16,6 +20,7 @@ namespace CodexPlugin
Metadata = metadata;
}
public String Id { get; }
public CodexInstance[] CodexInstances { get; }
public GethDeployment GethDeployment { get; }
public CodexContractsDeployment CodexContractsDeployment { get; }
@ -38,7 +43,10 @@ namespace CodexPlugin
public class DeploymentMetadata
{
public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace,
int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel,
int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI,
int blockMN)
{
Name = name;
StartUtc = startUtc;

5
Tests/CodexContinuousTests/ContinuousTestRunner.cs
View File

@ -27,10 +27,11 @@ namespace ContinuousTests
var startTime = DateTime.UtcNow;
var overviewLog = new LogSplitter(
new FixtureLog(logConfig, startTime, "Overview"),
new FixtureLog(logConfig, startTime, config.CodexDeployment.Id, "Overview"),
new ConsoleLog()
);
var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", "ContinuousTestRun");
var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", config.CodexDeployment.Id,
"ContinuousTestRun");
overviewLog.Log("Initializing...");

25
Tests/CodexContinuousTests/SingleTestRun.cs
View File

@ -25,7 +25,9 @@ namespace ContinuousTests
private readonly string testName;
private static int failureCount = 0;
public SingleTestRun(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
public SingleTestRun(EntryPointFactory entryPointFactory,
TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, TestHandle handle,
StartupChecker startupChecker, CancellationToken cancelToken, string deployId)
{
this.taskFactory = taskFactory;
this.config = config;
@ -34,8 +36,9 @@ namespace ContinuousTests
this.handle = handle;
this.cancelToken = cancelToken;
testName = handle.Test.GetType().Name;
fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, testName);
entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, fixtureLog);
fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName);
entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath,
config.CodexDeployment.Metadata.KubeNamespace, fixtureLog);
ApplyLogReplacements(fixtureLog, startupChecker);
nodes = CreateRandomNodes();
@ -80,6 +83,7 @@ namespace ContinuousTests
{
fixtureLog.Delete();
}
resultHandler(true);
}
catch (Exception ex)
@ -114,6 +118,7 @@ namespace ContinuousTests
{
effectiveStart = config.CodexDeployment.Metadata.StartUtc.Subtract(TimeSpan.FromSeconds(30));
}
var effectiveEnd = DateTime.UtcNow;
var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog);
@ -122,14 +127,17 @@ namespace ContinuousTests
var container = node.Container;
var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
var openingLine = $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().id}";
elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd, openingLine);
var openingLine =
$"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().id}";
elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
effectiveEnd, openingLine);
}
}
private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
{
foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
foreach (var replacement in startupChecker.LogReplacements)
fixtureLog.AddStringReplace(replacement.From, replacement.To);
}
private void RunTestMoments()
@ -161,9 +169,11 @@ namespace ContinuousTests
{
ThrowFailTest();
}
return;
}
}
fixtureLog.Log("Test run has been cancelled.");
}
@ -193,6 +203,7 @@ namespace ContinuousTests
result.Add("testname", testName);
result.Add("message", message);
result.Add("involvedpods", string.Join(",", nodes.Select(n => n.GetName())));
result.Add("involvedpodnames", string.Join(",", nodes.Select(n => n.GetPodInfo().Name)));
var error = message.Split(Environment.NewLine).First();
if (error.Contains(":")) error = error.Substring(1 + error.LastIndexOf(":"));
@ -222,6 +233,7 @@ namespace ContinuousTests
{
return UnpackException(a.InnerExceptions.First());
}
if (exception is TargetInvocationException t)
{
return UnpackException(t.InnerException!);
@ -294,6 +306,7 @@ namespace ContinuousTests
{
result[i] = containers.PickOneRandom();
}
return result;
}
}

25
Tests/CodexContinuousTests/StartupChecker.cs
View File

@ -23,7 +23,8 @@ namespace ContinuousTests
public void Check()
{
var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, "StartupChecks");
var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id,
"StartupChecks");
log.Log("Starting continuous test run...");
IncludeDeploymentConfiguration(log);
log.Log("Checking configuration...");
@ -46,13 +47,15 @@ namespace ContinuousTests
{
var podInfo = workflow.GetPodInfo(container);
log.Log($"Codex environment variables for '{container.Name}':");
log.Log($"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
log.Log(
$"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
$"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
var codexVars = container.Recipe.EnvVars;
foreach (var vars in codexVars) log.Log(vars.ToString());
log.Log("");
}
}
log.Log($"Deployment metadata: {JsonConvert.SerializeObject(deployment.Metadata)}");
log.Log("");
}
@ -64,6 +67,7 @@ namespace ContinuousTests
{
throw new Exception("Unable to find any tests.");
}
foreach (var test in tests)
{
cancelToken.ThrowIfCancellationRequested();
@ -87,7 +91,8 @@ namespace ContinuousTests
private void CheckCodexNodes(BaseLog log, Configuration config)
{
var nodes = entryPoint.CreateInterface().WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
var nodes = entryPoint.CreateInterface()
.WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
var pass = true;
foreach (var n in nodes)
{
@ -106,6 +111,7 @@ namespace ContinuousTests
pass = false;
}
}
if (!pass)
{
throw new Exception("Not all codex nodes responded.");
@ -126,6 +132,7 @@ namespace ContinuousTests
{
return false;
}
return true;
}
@ -145,7 +152,8 @@ namespace ContinuousTests
propertyName: nameof(ContinuousTest.CustomK8sNamespace));
}
private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors, Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors,
Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
{
foreach (var test in tests)
{
@ -155,7 +163,8 @@ namespace ContinuousTests
if (duplicates.Any())
{
duplicates.Add(test);
errors.Add($"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
errors.Add(
$"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
return;
}
}
@ -170,11 +179,13 @@ namespace ContinuousTests
{
if (test.RequiredNumberOfNodes < 1)
{
errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
errors.Add(
$"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
}
else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexInstances.Length)
{
errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}");
errors.Add(
$"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}");
}
}
}

3
Tests/CodexContinuousTests/TestLoop.cs
View File

@ -77,7 +77,8 @@ namespace ContinuousTests
{
var test = (ContinuousTest)Activator.CreateInstance(testType)!;
var handle = new TestHandle(test);
var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, statusLog, handle, startupChecker, cancelToken);
var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, statusLog, handle,
startupChecker, cancelToken, config.CodexDeployment.Id);
runFinishedHandle.Reset();
run.Run(runFinishedHandle, result =>

1
Tests/CodexTests/AutoBootstrapDistTest.cs
View File

@ -23,5 +23,6 @@ namespace CodexTests
}
protected ICodexNode? BootstrapNode { get; private set; }
}
}

9
Tests/DistTestCore/DistTest.cs
View File

@ -20,16 +20,19 @@ namespace DistTestCore
private readonly object lifecycleLock = new object();
private readonly EntryPoint globalEntryPoint;
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
private readonly string deployId;
public DistTest()
{
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
deployId = NameUtils.MakeDeployId();
var logConfig = configuration.GetLogConfig();
var startTime = DateTime.UtcNow;
fixtureLog = new FixtureLog(logConfig, startTime);
statusLog = new StatusLog(logConfig, startTime, "dist-tests");
fixtureLog = new FixtureLog(logConfig, startTime, deployId);
statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId);
globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
@ -181,7 +184,7 @@ namespace DistTestCore
lock (lifecycleLock)
{
var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace, deployId);
lifecycles.Add(testName, lifecycle);
LifecycleStart(lifecycle);
}

9
Tests/DistTestCore/Logs/BaseTestLog.cs
View File

@ -5,13 +5,18 @@ namespace DistTestCore.Logs
public abstract class BaseTestLog : BaseLog
{
private bool hasFailed;
private readonly string deployId;
protected BaseTestLog(string deployId)
{
this.deployId = deployId;
}
public void WriteLogTag()
{
var runId = NameUtils.GetRunId();
var category = NameUtils.GetCategoryName();
var name = NameUtils.GetTestMethodName();
LogFile.WriteRaw($"{runId} {category} {name}");
LogFile.WriteRaw($"{deployId} {category} {name}");
}
public void MarkAsFailed()

6
Tests/DistTestCore/Logs/FixtureLog.cs
View File

@ -5,15 +5,17 @@ namespace DistTestCore.Logs
public class FixtureLog : BaseTestLog
{
private readonly string fullName;
private readonly string deployId;
public FixtureLog(LogConfig config, DateTime start, string name = "")
public FixtureLog(LogConfig config, DateTime start, string deployId, string name = "") : base(deployId)
{
this.deployId = deployId;
fullName = NameUtils.GetFixtureFullName(config, start, name);
}
public TestLog CreateTestLog(string name = "")
{
return new TestLog(fullName, name);
return new TestLog(fullName, deployId, name);
}
public void DeleteFolder()

8
Tests/DistTestCore/Logs/StatusLog.cs
View File

@ -6,16 +6,18 @@ namespace DistTestCore.Logs
{
public class StatusLog
{
private readonly object fileLock = new object();
private readonly object fileLock = new();
private readonly string deployId;
private readonly string fullName;
private readonly string fixtureName;
private readonly string testType;
public StatusLog(LogConfig config, DateTime start, string testType, string name = "")
public StatusLog(LogConfig config, DateTime start, string testType, string deployId, string name = "")
{
fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
fixtureName = NameUtils.GetRawFixtureName();
this.testType = testType;
this.deployId = deployId;
}
public void ConcludeTest(string resultStatus, TimeSpan testDuration, Dictionary<string, string> data)
@ -26,7 +28,7 @@ namespace DistTestCore.Logs
public void ConcludeTest(string resultStatus, string testDuration, Dictionary<string, string> data)
{
data.Add("timestamp", DateTime.UtcNow.ToString("o"));
data.Add("runid", NameUtils.GetRunId());
data.Add("deployid", deployId);
data.Add("status", resultStatus);
data.Add("category", NameUtils.GetCategoryName());
data.Add("fixturename", fixtureName);

5
Tests/DistTestCore/Logs/TestLog.cs
View File

@ -2,12 +2,11 @@
{
public class TestLog : BaseTestLog
{
private readonly string methodName;
private readonly string fullName;
public TestLog(string folder, string name = "")
public TestLog(string folder, string deployId, string name = "") : base(deployId)
{
methodName = NameUtils.GetTestMethodName(name);
var methodName = NameUtils.GetTestMethodName(name);
fullName = Path.Combine(folder, methodName);
Log($"*** Begin: {methodName}");

12
Tests/DistTestCore/NameUtils.cs
View File

@ -5,14 +5,6 @@ namespace DistTestCore
{
public static class NameUtils
{
private static readonly string defaultRunId;
static NameUtils()
{
var now = DateTime.UtcNow;
defaultRunId = now.ToString("yyyyMMdd-hhmmss");
}
public static string GetTestMethodName(string name = "")
{
if (!string.IsNullOrEmpty(name)) return name;
@ -48,9 +40,9 @@ namespace DistTestCore
return GetEnvVar("TESTID", "EnvVar-TESTID-NotSet");
}
public static string GetRunId()
public static string MakeDeployId()
{
return GetEnvVar("RUNID", defaultRunId);
return DateTime.UtcNow.ToString("yyyyMMdd-hhmmss");
}
private static string GetEnvVar(string name, string defaultValue)

8
Tests/DistTestCore/TestLifecycle.cs
View File

@ -13,9 +13,10 @@ namespace DistTestCore
private const string TestsType = "dist-tests";
private readonly EntryPoint entryPoint;
private readonly Dictionary<string, string> metadata;
private readonly List<RunningContainers> runningContainers = new List<RunningContainers>();
private readonly List<RunningContainers> runningContainers = new();
private readonly string deployId;
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
{
Log = log;
Configuration = configuration;
@ -25,6 +26,7 @@ namespace DistTestCore
entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
metadata = entryPoint.GetPluginMetadata();
CoreInterface = entryPoint.CreateInterface();
this.deployId = deployId;
log.WriteLogTag();
}
@ -76,7 +78,7 @@ namespace DistTestCore
public void OnContainerRecipeCreated(ContainerRecipe recipe)
{
recipe.PodLabels.Add("tests-type", TestsType);
recipe.PodLabels.Add("runid", NameUtils.GetRunId());
recipe.PodLabels.Add("deployid", deployId);
recipe.PodLabels.Add("testid", NameUtils.GetTestId());
recipe.PodLabels.Add("category", NameUtils.GetCategoryName());
recipe.PodLabels.Add("fixturename", NameUtils.GetRawFixtureName());

4
Tools/CodexNetDeployer/Configuration.cs
View File

@ -1,5 +1,6 @@
using ArgsUniform;
using CodexPlugin;
using DistTestCore;
namespace CodexNetDeployer
{
@ -17,6 +18,9 @@ namespace CodexNetDeployer
[Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
public string KubeNamespace { get; set; } = string.Empty;
[Uniform("deploy-id", "di", "DEPLOYID", false, "ID of the deployment. (default) to current time)")]
public string DeployId { get; set; } = NameUtils.MakeDeployId();
[Uniform("deploy-file", "df", "DEPLOYFILE", false, "Output deployment JSON file that will be written. Defaults to 'codex-deployment.json'.")]
public string DeployFile { get; set; } = "codex-deployment.json";

15
Tools/CodexNetDeployer/Deployer.cs
View File

@ -45,6 +45,7 @@ namespace CodexNetDeployer
Console.CursorLeft = longestKey + 5;
Console.WriteLine($"= {entry.Value}");
}
Log("");
}
@ -84,7 +85,8 @@ namespace CodexNetDeployer
var discordBotContainer = DeployDiscordBot(ci, gethDeployment, contractsDeployment);
return new CodexDeployment(codexInstances, gethDeployment, contractsDeployment, metricsService, discordBotContainer, CreateMetadata(startUtc));
return new CodexDeployment(codexInstances, gethDeployment, contractsDeployment, metricsService,
discordBotContainer, CreateMetadata(startUtc), config.DeployId);
}
private EntryPoint CreateEntryPoint(ILog log)
@ -98,7 +100,7 @@ namespace CodexNetDeployer
kubernetesNamespace: config.KubeNamespace);
var result = new EntryPoint(log, configuration, string.Empty, new FastHttpTimeSet());
configuration.Hooks = new K8sHook(config.TestsTypePodLabel, result.GetPluginMetadata());
configuration.Hooks = new K8sHook(config.TestsTypePodLabel, config.DeployId, result.GetPluginMetadata());
return result;
}
@ -120,7 +122,8 @@ namespace CodexNetDeployer
});
}
private RunningContainers? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment, CodexContractsDeployment contractsDeployment)
private RunningContainers? DeployDiscordBot(CoreInterface ci, GethDeployment gethDeployment,
CodexContractsDeployment contractsDeployment)
{
if (!config.DeployDiscordBot) return null;
Log("Deploying Discord bot...");
@ -201,7 +204,8 @@ namespace CodexNetDeployer
foreach (var startResult in startResults)
{
var watcher = startResult.CodexNode.CrashWatcher;
if (watcher == null) throw new Exception("Expected each CodexNode container to be created with a crash-watcher.");
if (watcher == null)
throw new Exception("Expected each CodexNode container to be created with a crash-watcher.");
if (watcher.HasContainerCrashed()) crashes.Add(startResult.CodexNode.Container);
}
@ -211,7 +215,8 @@ namespace CodexNetDeployer
}
else
{
Log($"Check failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
Log(
$"Check failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
throw new Exception("Deployment failed: One or more containers crashed.");
}
}

6
Tools/CodexNetDeployer/K8sHook.cs
View File

@ -8,11 +8,13 @@ namespace CodexNetDeployer
public class K8sHook : IK8sHooks
{
private readonly string testsTypeLabel;
private readonly string deployId;
private readonly Dictionary<string, string> metadata;
public K8sHook(string testsTypeLabel, Dictionary<string, string> metadata)
public K8sHook(string testsTypeLabel, string deployId, Dictionary<string, string> metadata)
{
this.testsTypeLabel = testsTypeLabel;
this.deployId = deployId;
this.metadata = metadata;
}
@ -27,7 +29,7 @@ namespace CodexNetDeployer
public void OnContainerRecipeCreated(ContainerRecipe recipe)
{
recipe.PodLabels.Add("tests-type", testsTypeLabel);
recipe.PodLabels.Add("runid", NameUtils.GetRunId());
recipe.PodLabels.Add("deployid", deployId);
recipe.PodLabels.Add("testid", NameUtils.GetTestId());
foreach (var pair in metadata)