status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cleanup some workflow/lifecycle madness

This commit is contained in:
benbierens 2023-08-10 10:58:18 +02:00
parent 9a63a82a49
commit 17673e8fa5
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
17 changed files with 56 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CodexNetDeployer/CodexNodeStarter.cs
View File

@ -8,16 +8,14 @@ namespace CodexNetDeployer
public class CodexNodeStarter
{
private readonly Configuration config;
private readonly WorkflowCreator workflowCreator;
private readonly TestLifecycle lifecycle;
private readonly GethStartResult gethResult;
private string bootstrapSpr = "";
private int validatorsLeft;
public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
{
this.config = config;
this.workflowCreator = workflowCreator;
this.lifecycle = lifecycle;
this.gethResult = gethResult;
validatorsLeft = numberOfValidators;
@ -26,7 +24,7 @@ namespace CodexNetDeployer
public RunningContainer? Start(int i)
{
Console.Write($" - {i} = ");
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var workflowStartup = new StartupConfig();
workflowStartup.Add(gethResult);
workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));

25
CodexNetDeployer/Deployer.cs
View File

@ -21,7 +21,7 @@ namespace CodexNetDeployer
public CodexDeployment Deploy()
{
Log("Initializing...");
var (workflowCreator, lifecycle) = CreateFacilities();
var lifecycle = CreateTestLifecycle();
Log("Preparing configuration...");
// We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
@ -30,7 +30,7 @@ namespace CodexNetDeployer
setup.MetricsEnabled = config.RecordMetrics;
Log("Creating Geth instance and deploying contracts...");
var gethStarter = new GethStarter(lifecycle, workflowCreator);
var gethStarter = new GethStarter(lifecycle);
var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
Log("Geth started. Codex contracts deployed.");
@ -44,7 +44,7 @@ namespace CodexNetDeployer
Log("Starting Codex nodes...");
// Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value);
var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
var codexContainers = new List<RunningContainer>();
for (var i = 0; i < config.NumberOfCodexNodes; i++)
{
@ -57,7 +57,7 @@ namespace CodexNetDeployer
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
}
private (WorkflowCreator, TestLifecycle) CreateFacilities()
private TestLifecycle CreateTestLifecycle()
{
var kubeConfig = GetKubeConfig(config.KubeConfigFile);
@ -68,22 +68,11 @@ namespace CodexNetDeployer
logDebug: false,
dataFilesPath: "notUsed",
codexLogLevel: config.CodexLogLevel,
runnerLocation: config.RunnerLocation
runnerLocation: config.RunnerLocation,
k8sNamespacePrefix: config.KubeNamespace
);
var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: config.KubeNamespace,
kubeConfigFile: kubeConfig,
operationTimeout: timeset.K8sOperationTimeout(),
retryDelay: timeset.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(log, kubeFlowConfig,
testNamespacePostfix: string.Empty,
testsType: config.TestsTypePodLabel);
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
return (workflowCreator, lifecycle);
return new TestLifecycle(log, lifecycleConfig, timeset);
}
private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)

2
CodexNetDownloader/Program.cs
View File

@ -25,7 +25,7 @@ public class Program
if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
var k8sFactory = new K8sFactory();
var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
foreach (var container in config.CodexDeployment.CodexContainers)
{

6
ContinuousTests/ContinuousTestRunner.cs
View File

@ -60,8 +60,8 @@ namespace ContinuousTests
if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
workflowCreator.CreateWorkflow().DeleteTestResources();
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
}
private void StartLogDownloader(TaskFactory taskFactory)
@ -71,7 +71,7 @@ namespace ContinuousTests
var path = Path.Combine(config.LogPath, "containers");
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
taskFactory.Run(downloader.Run);

20
ContinuousTests/K8sFactory.cs
View File

@ -1,13 +1,12 @@
using DistTestCore.Codex;
using DistTestCore;
using KubernetesWorkflow;
using Logging;
namespace ContinuousTests
{
public class K8sFactory
{
public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
{
var kubeConfig = GetKubeConfig(kubeConfigFile);
var lifecycleConfig = new DistTestCore.Configuration
@ -17,22 +16,11 @@ namespace ContinuousTests
logDebug: false,
dataFilesPath: dataFilePath,
codexLogLevel: CodexLogLevel.Debug,
runnerLocation: runnerLocation
runnerLocation: runnerLocation,
k8sNamespacePrefix: customNamespace
);
var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: customNamespace,
kubeConfigFile: kubeConfig,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(log, kubeFlowConfig,
testNamespacePostfix: string.Empty,
testsType: "continuous");
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
return (workflowCreator, lifecycle);
return new TestLifecycle(log, lifecycleConfig, timeSet);
}
private static string? GetKubeConfig(string kubeConfigFile)

8
ContinuousTests/NodeRunner.cs
View File

@ -40,8 +40,8 @@ namespace ContinuousTests
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
{
var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow();
var lifecycle = CreateTestLifecycle();
var flow = lifecycle.WorkflowCreator.CreateWorkflow();
try
{
@ -89,9 +89,9 @@ namespace ContinuousTests
}
}
private (WorkflowCreator, TestLifecycle) CreateFacilities()
private TestLifecycle CreateTestLifecycle()
{
return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
}
}
}

4
ContinuousTests/SingleTestRun.cs
View File

@ -138,7 +138,7 @@ namespace ContinuousTests
private void DownloadClusterLogs()
{
var k8sFactory = new K8sFactory();
var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
foreach (var container in config.CodexDeployment.CodexContainers)
{
@ -221,7 +221,7 @@ namespace ContinuousTests
private DistTestCore.Configuration CreateFileManagerConfiguration()
{
return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
CodexLogLevel.Error, config.RunnerLocation);
CodexLogLevel.Error, config.RunnerLocation, string.Empty);
}
}
}

7
DistTestCore/BaseStarter.cs
View File

@ -1,18 +1,15 @@
using KubernetesWorkflow;
using Logging;
using Logging;
namespace DistTestCore
{
public class BaseStarter
{
protected readonly TestLifecycle lifecycle;
protected readonly WorkflowCreator workflowCreator;
private Stopwatch? stopwatch;
public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
public BaseStarter(TestLifecycle lifecycle)
{
this.lifecycle = lifecycle;
this.workflowCreator = workflowCreator;
}
protected void LogStart(string msg)

6
DistTestCore/CodexStarter.cs
View File

@ -8,8 +8,8 @@ namespace DistTestCore
{
public class CodexStarter : BaseStarter
{
public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public CodexStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -121,7 +121,7 @@ namespace DistTestCore
private StartupWorkflow CreateWorkflow()
{
return workflowCreator.CreateWorkflow();
return lifecycle.WorkflowCreator.CreateWorkflow();
}
private void LogSeparator()

7
DistTestCore/Configuration.cs
View File

@ -12,6 +12,7 @@ namespace DistTestCore
private readonly string dataFilesPath;
private readonly CodexLogLevel codexLogLevel;
private readonly TestRunnerLocation runnerLocation;
private readonly string k8sNamespacePrefix;
public Configuration()
{
@ -21,9 +22,10 @@ namespace DistTestCore
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
k8sNamespacePrefix = "ct-";
}
public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix)
{
this.kubeConfigFile = kubeConfigFile;
this.logPath = logPath;
@ -31,12 +33,13 @@ namespace DistTestCore
this.dataFilesPath = dataFilesPath;
this.codexLogLevel = codexLogLevel;
this.runnerLocation = runnerLocation;
this.k8sNamespacePrefix = k8sNamespacePrefix;
}
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{
return new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: "ct-",
k8sNamespacePrefix: k8sNamespacePrefix,
kubeConfigFile: kubeConfigFile,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay()

11
DistTestCore/GethStarter.cs
View File

@ -1,5 +1,4 @@
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore
{
@ -8,13 +7,13 @@ namespace DistTestCore
private readonly MarketplaceNetworkCache marketplaceNetworkCache;
private readonly GethCompanionNodeStarter companionNodeStarter;
public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public GethStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
marketplaceNetworkCache = new MarketplaceNetworkCache(
new GethBootstrapNodeStarter(lifecycle, workflowCreator),
new CodexContractsStarter(lifecycle, workflowCreator));
companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
new GethBootstrapNodeStarter(lifecycle),
new CodexContractsStarter(lifecycle));
companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
}
public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)

6
DistTestCore/Marketplace/CodexContractsStarter.cs
View File

@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace
public class CodexContractsStarter : BaseStarter
{
public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public CodexContractsStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace
{
LogStart("Deploying Codex Marketplace...");
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);

6
DistTestCore/Marketplace/GethBootstrapNodeStarter.cs
View File

@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace
{
public class GethBootstrapNodeStarter : BaseStarter
{
public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public GethBootstrapNodeStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace
LogStart("Starting Geth bootstrap node...");
var startupConfig = CreateBootstrapStartupConfig();
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
var bootstrapContainer = containers.Containers[0];

6
DistTestCore/Marketplace/GethCompanionNodeStarter.cs
View File

@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace
{
private int companionAccountIndex = 0;
public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public GethCompanionNodeStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace
var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
var container = containers.Containers[0];

6
DistTestCore/PrometheusStarter.cs
View File

@ -7,8 +7,8 @@ namespace DistTestCore
{
public class PrometheusStarter : BaseStarter
{
public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public PrometheusStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -18,7 +18,7 @@ namespace DistTestCore
var startupConfig = new StartupConfig();
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");

14
DistTestCore/TestLifecycle.cs
View File

@ -11,20 +11,17 @@ namespace DistTestCore
private readonly DateTime testStart;
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
: this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests"))
{
}
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
{
Log = log;
Configuration = configuration;
TimeSet = timeSet;
WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests");
FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this, workflowCreator);
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator);
CodexStarter = new CodexStarter(this);
PrometheusStarter = new PrometheusStarter(this);
GethStarter = new GethStarter(this);
testStart = DateTime.UtcNow;
CodexVersion = null;
@ -34,6 +31,7 @@ namespace DistTestCore
public BaseLog Log { get; }
public Configuration Configuration { get; }
public ITimeSet TimeSet { get; }
public WorkflowCreator WorkflowCreator { get; }
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }

1
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -27,7 +27,6 @@ namespace KubernetesWorkflow
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
{
var podLabels = new PodLabels();
podLabels.Add("codex-test-node", "dist-test-" + numberSource.WorkflowNumber);
podLabels.Add("runid", NameUtils.GetRunId());
podLabels.Add("tests-type", testsType);
podLabels.Add("app", recipeFactory.AppName);