diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 9478a98..0a3b373 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -8,16 +8,14 @@ namespace CodexNetDeployer public class CodexNodeStarter { private readonly Configuration config; - private readonly WorkflowCreator workflowCreator; private readonly TestLifecycle lifecycle; private readonly GethStartResult gethResult; private string bootstrapSpr = ""; private int validatorsLeft; - public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators) + public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators) { this.config = config; - this.workflowCreator = workflowCreator; this.lifecycle = lifecycle; this.gethResult = gethResult; validatorsLeft = numberOfValidators; @@ -26,7 +24,7 @@ namespace CodexNetDeployer public RunningContainer? Start(int i) { Console.Write($" - {i} = "); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var workflowStartup = new StartupConfig(); workflowStartup.Add(gethResult); workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft)); diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index fc3c425..dda9cd8 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -21,7 +21,7 @@ namespace CodexNetDeployer public CodexDeployment Deploy() { Log("Initializing..."); - var (workflowCreator, lifecycle) = CreateFacilities(); + var lifecycle = CreateTestLifecycle(); Log("Preparing configuration..."); // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. @@ -30,7 +30,7 @@ namespace CodexNetDeployer setup.MetricsEnabled = config.RecordMetrics; Log("Creating Geth instance and deploying contracts..."); - var gethStarter = new GethStarter(lifecycle, workflowCreator); + var gethStarter = new GethStarter(lifecycle); var gethResults = gethStarter.BringOnlineMarketplaceFor(setup); Log("Geth started. Codex contracts deployed."); @@ -44,7 +44,7 @@ namespace CodexNetDeployer Log("Starting Codex nodes..."); // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. - var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value); + var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value); var codexContainers = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { @@ -57,7 +57,7 @@ namespace CodexNetDeployer return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata()); } - private (WorkflowCreator, TestLifecycle) CreateFacilities() + private TestLifecycle CreateTestLifecycle() { var kubeConfig = GetKubeConfig(config.KubeConfigFile); @@ -68,22 +68,11 @@ namespace CodexNetDeployer logDebug: false, dataFilesPath: "notUsed", codexLogLevel: config.CodexLogLevel, - runnerLocation: config.RunnerLocation + runnerLocation: config.RunnerLocation, + k8sNamespacePrefix: config.KubeNamespace ); - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: config.KubeNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: timeset.K8sOperationTimeout(), - retryDelay: timeset.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, - testNamespacePostfix: string.Empty, - testsType: config.TestsTypePodLabel); - - var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); - - return (workflowCreator, lifecycle); + return new TestLifecycle(log, lifecycleConfig, timeset); } private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs index f52a1d2..195e96b 100644 --- a/CodexNetDownloader/Program.cs +++ b/CodexNetDownloader/Program.cs @@ -25,7 +25,7 @@ public class Program if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath); var k8sFactory = new K8sFactory(); - var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); foreach (var container in config.CodexDeployment.CodexContainers) { diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 985707a..35ff486 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -60,8 +60,8 @@ namespace ContinuousTests if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); - var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); - workflowCreator.CreateWorkflow().DeleteTestResources(); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); + lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources(); } private void StartLogDownloader(TaskFactory taskFactory) @@ -71,7 +71,7 @@ namespace ContinuousTests var path = Path.Combine(config.LogPath, "containers"); if (!Directory.Exists(path)) Directory.CreateDirectory(path); - var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken); taskFactory.Run(downloader.Run); diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index fc3a6c0..7695a76 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -1,13 +1,12 @@ using DistTestCore.Codex; using DistTestCore; -using KubernetesWorkflow; using Logging; namespace ContinuousTests { public class K8sFactory { - public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) + public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) { var kubeConfig = GetKubeConfig(kubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration @@ -17,22 +16,11 @@ namespace ContinuousTests logDebug: false, dataFilesPath: dataFilePath, codexLogLevel: CodexLogLevel.Debug, - runnerLocation: runnerLocation + runnerLocation: runnerLocation, + k8sNamespacePrefix: customNamespace ); - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: customNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, - testNamespacePostfix: string.Empty, - testsType: "continuous"); - - var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator); - - return (workflowCreator, lifecycle); + return new TestLifecycle(log, lifecycleConfig, timeSet); } private static string? GetKubeConfig(string kubeConfigFile) diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 02116ee..4bb335e 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -40,8 +40,8 @@ namespace ContinuousTests public void RunNode(CodexAccess bootstrapNode, Action operation, TestToken mintTestTokens) { - var (workflowCreator, lifecycle) = CreateFacilities(); - var flow = workflowCreator.CreateWorkflow(); + var lifecycle = CreateTestLifecycle(); + var flow = lifecycle.WorkflowCreator.CreateWorkflow(); try { @@ -89,9 +89,9 @@ namespace ContinuousTests } } - private (WorkflowCreator, TestLifecycle) CreateFacilities() + private TestLifecycle CreateTestLifecycle() { - return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); + return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); } } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index b065c77..cff325e 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -138,7 +138,7 @@ namespace ContinuousTests private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); - var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); foreach (var container in config.CodexDeployment.CodexContainers) { @@ -221,7 +221,7 @@ namespace ContinuousTests private DistTestCore.Configuration CreateFileManagerConfiguration() { return new DistTestCore.Configuration(null, string.Empty, false, dataFolder, - CodexLogLevel.Error, config.RunnerLocation); + CodexLogLevel.Error, config.RunnerLocation, string.Empty); } } } diff --git a/DistTestCore/BaseStarter.cs b/DistTestCore/BaseStarter.cs index 83a6b8d..4d10643 100644 --- a/DistTestCore/BaseStarter.cs +++ b/DistTestCore/BaseStarter.cs @@ -1,18 +1,15 @@ -using KubernetesWorkflow; -using Logging; +using Logging; namespace DistTestCore { public class BaseStarter { protected readonly TestLifecycle lifecycle; - protected readonly WorkflowCreator workflowCreator; private Stopwatch? stopwatch; - public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + public BaseStarter(TestLifecycle lifecycle) { this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; } protected void LogStart(string msg) diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 690ee94..a1e38b7 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -8,8 +8,8 @@ namespace DistTestCore { public class CodexStarter : BaseStarter { - public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public CodexStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -121,7 +121,7 @@ namespace DistTestCore private StartupWorkflow CreateWorkflow() { - return workflowCreator.CreateWorkflow(); + return lifecycle.WorkflowCreator.CreateWorkflow(); } private void LogSeparator() diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 03514cd..daccad6 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -12,6 +12,7 @@ namespace DistTestCore private readonly string dataFilesPath; private readonly CodexLogLevel codexLogLevel; private readonly TestRunnerLocation runnerLocation; + private readonly string k8sNamespacePrefix; public Configuration() { @@ -21,9 +22,10 @@ namespace DistTestCore dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); + k8sNamespacePrefix = "ct-"; } - public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) + public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix) { this.kubeConfigFile = kubeConfigFile; this.logPath = logPath; @@ -31,12 +33,13 @@ namespace DistTestCore this.dataFilesPath = dataFilesPath; this.codexLogLevel = codexLogLevel; this.runnerLocation = runnerLocation; + this.k8sNamespacePrefix = k8sNamespacePrefix; } public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet) { return new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: "ct-", + k8sNamespacePrefix: k8sNamespacePrefix, kubeConfigFile: kubeConfigFile, operationTimeout: timeSet.K8sOperationTimeout(), retryDelay: timeSet.WaitForK8sServiceDelay() diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 3b9a9b1..578cc30 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -1,5 +1,4 @@ using DistTestCore.Marketplace; -using KubernetesWorkflow; namespace DistTestCore { @@ -8,13 +7,13 @@ namespace DistTestCore private readonly MarketplaceNetworkCache marketplaceNetworkCache; private readonly GethCompanionNodeStarter companionNodeStarter; - public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethStarter(TestLifecycle lifecycle) + : base(lifecycle) { marketplaceNetworkCache = new MarketplaceNetworkCache( - new GethBootstrapNodeStarter(lifecycle, workflowCreator), - new CodexContractsStarter(lifecycle, workflowCreator)); - companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); + new GethBootstrapNodeStarter(lifecycle), + new CodexContractsStarter(lifecycle)); + companionNodeStarter = new GethCompanionNodeStarter(lifecycle); } public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup) diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index c41df53..a1db57f 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace public class CodexContractsStarter : BaseStarter { - public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public CodexContractsStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace { LogStart("Deploying Codex Marketplace..."); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]); var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 56296ff..d1ebb54 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeStarter : BaseStarter { - public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethBootstrapNodeStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace LogStart("Starting Geth bootstrap node..."); var startupConfig = CreateBootstrapStartupConfig(); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); var bootstrapContainer = containers.Containers[0]; diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index 2008e9c..6759e7b 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace { private int companionAccountIndex = 0; - public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethCompanionNodeStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config)); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 6ef2e57..1804b77 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -7,8 +7,8 @@ namespace DistTestCore { public class PrometheusStarter : BaseStarter { - public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public PrometheusStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -18,7 +18,7 @@ namespace DistTestCore var startupConfig = new StartupConfig(); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers()))); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 4c674cf..26c0a5a 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -11,20 +11,17 @@ namespace DistTestCore private readonly DateTime testStart; public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) - : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests")) - { - } - - public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) { Log = log; Configuration = configuration; TimeSet = timeSet; + WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests"); + FileManager = new FileManager(Log, configuration); - CodexStarter = new CodexStarter(this, workflowCreator); - PrometheusStarter = new PrometheusStarter(this, workflowCreator); - GethStarter = new GethStarter(this, workflowCreator); + CodexStarter = new CodexStarter(this); + PrometheusStarter = new PrometheusStarter(this); + GethStarter = new GethStarter(this); testStart = DateTime.UtcNow; CodexVersion = null; @@ -34,6 +31,7 @@ namespace DistTestCore public BaseLog Log { get; } public Configuration Configuration { get; } public ITimeSet TimeSet { get; } + public WorkflowCreator WorkflowCreator { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index a4b9df0..67f757f 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -27,7 +27,6 @@ namespace KubernetesWorkflow public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { var podLabels = new PodLabels(); - podLabels.Add("codex-test-node", "dist-test-" + numberSource.WorkflowNumber); podLabels.Add("runid", NameUtils.GetRunId()); podLabels.Add("tests-type", testsType); podLabels.Add("app", recipeFactory.AppName);