From 1e7470d476e9e9026d1d65d8d9f84ca7f693b6f9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 7 Aug 2023 15:07:20 +0200 Subject: [PATCH 01/10] Lays out the plan --- KubernetesWorkflow/K8sController.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index d679c8c..e8dc393 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -362,7 +362,13 @@ namespace KubernetesWorkflow private IDictionary GetSelector() { - return new Dictionary { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } }; + return new Dictionary + { + { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } + // tests-type=dist-tests + // app=codex + // runid=20230721-085043 + }; } private IDictionary GetRunnerNamespaceSelector() From 0259c30bb509b0d81c3e90ac1bd63393ed380a40 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 7 Aug 2023 15:51:44 +0200 Subject: [PATCH 02/10] Wires up pod labels --- CodexNetDeployer/Configuration.cs | 4 ++++ CodexNetDeployer/Deployer.cs | 5 ++++- ContinuousTests/K8sFactory.cs | 5 ++++- DistTestCore/Codex/CodexContainerRecipe.cs | 1 + DistTestCore/DistTest.cs | 2 +- .../CodexContractsContainerRecipe.cs | 8 ++----- .../Marketplace/GethContainerRecipe.cs | 8 ++----- .../Metrics/PrometheusContainerRecipe.cs | 8 ++----- DistTestCore/TestLifecycle.cs | 2 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 1 + KubernetesWorkflow/K8sController.cs | 12 ++++------ KubernetesWorkflow/PodLabels.cs | 17 ++++++++++++++ KubernetesWorkflow/StartupWorkflow.cs | 22 +++++++++++++++---- KubernetesWorkflow/WorkflowCreator.cs | 10 +++++---- Logging/NameUtils.cs | 2 +- 15 files changed, 68 insertions(+), 39 deletions(-) create mode 100644 KubernetesWorkflow/PodLabels.cs diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index b030618..955084a 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -46,6 +46,10 @@ namespace CodexNetDeployer [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] public bool RecordMetrics { get; set; } = false; + + [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " + + "set this option to override the label value.")] + public string TestsTypePodLabel { get; set; } = "continuous"; public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 4bfa25d..fc3c425 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -77,7 +77,10 @@ namespace CodexNetDeployer operationTimeout: timeset.K8sOperationTimeout(), retryDelay: timeset.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); + var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, + testNamespacePostfix: string.Empty, + testsType: config.TestsTypePodLabel); + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); return (workflowCreator, lifecycle); diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index 0704e9f..fc3a6c0 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -26,7 +26,10 @@ namespace ContinuousTests operationTimeout: timeSet.K8sOperationTimeout(), retryDelay: timeSet.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); + var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, + testNamespacePostfix: string.Empty, + testsType: "continuous"); + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator); return (workflowCreator, lifecycle); diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index ea6f187..c600c01 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -14,6 +14,7 @@ namespace DistTestCore.Codex public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); + public override string AppName => "codex"; public override string Image { get; } public CodexContainerRecipe() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 6768618..e05c566 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -52,7 +52,7 @@ namespace DistTestCore { Stopwatch.Measure(fixtureLog, "Global setup", () => { - var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet())); + var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), "dist-tests"); wc.CreateWorkflow().DeleteAllResources(); }); } diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index d4bc5d1..92a6c81 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -7,12 +7,8 @@ namespace DistTestCore.Marketplace public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; - public override string Image { get; } - - public CodexContractsContainerRecipe() - { - Image = "codexstorage/dist-tests-codex-contracts-eth:sha-9a83699"; - } + public override string AppName => "codex-contracts"; + public override string Image => "codexstorage/dist-tests-codex-contracts-eth:sha-9a83699"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 6dc62e9..4ca7895 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -10,12 +10,8 @@ namespace DistTestCore.Marketplace public const string DiscoveryPortTag = "disc_port"; public const string AccountsFilename = "accounts.csv"; - public override string Image { get; } - - public GethContainerRecipe() - { - Image = "codexstorage/dist-tests-geth:sha-b788a2d"; - } + public override string AppName => "geth"; + public override string Image => "codexstorage/dist-tests-geth:sha-b788a2d"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs index b702222..e51fdc5 100644 --- a/DistTestCore/Metrics/PrometheusContainerRecipe.cs +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -4,12 +4,8 @@ namespace DistTestCore.Metrics { public class PrometheusContainerRecipe : ContainerRecipeFactory { - public override string Image { get; } - - public PrometheusContainerRecipe() - { - Image = "codexstorage/dist-tests-prometheus:sha-f97d7fd"; - } + public override string AppName => "prometheus"; + public override string Image => "codexstorage/dist-tests-prometheus:sha-f97d7fd"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index f33b238..4c674cf 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -11,7 +11,7 @@ namespace DistTestCore private readonly DateTime testStart; public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) - : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet))) + : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests")) { } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 340ed2d..470bd42 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -27,6 +27,7 @@ return recipe; } + public abstract string AppName { get; } public abstract string Image { get; } protected int ContainerNumber { get; private set; } = 0; protected int Index { get; private set; } = 0; diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index e8dc393..5569b07 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -11,14 +11,16 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; + private readonly PodLabels podLabels; private readonly K8sClient client; - public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace) + public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace, PodLabels podLabels) { this.log = log; this.cluster = cluster; this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; + this.podLabels = podLabels; client = new K8sClient(cluster.GetK8sClientConfig()); K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace; @@ -362,13 +364,7 @@ namespace KubernetesWorkflow private IDictionary GetSelector() { - return new Dictionary - { - { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } - // tests-type=dist-tests - // app=codex - // runid=20230721-085043 - }; + return podLabels.GetLabels(); } private IDictionary GetRunnerNamespaceSelector() diff --git a/KubernetesWorkflow/PodLabels.cs b/KubernetesWorkflow/PodLabels.cs new file mode 100644 index 0000000..8c444bf --- /dev/null +++ b/KubernetesWorkflow/PodLabels.cs @@ -0,0 +1,17 @@ +namespace KubernetesWorkflow +{ + public class PodLabels + { + private readonly Dictionary labels = new Dictionary(); + + public void Add(string key, string value) + { + labels.Add(key, value.ToLowerInvariant()); + } + + internal Dictionary GetLabels() + { + return labels; + } + } +} diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index c7e3bfb..a4b9df0 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -10,19 +10,28 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownK8SPods; private readonly string testNamespace; + private readonly string testsType; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace) + internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, string testsType) { this.log = log; this.numberSource = numberSource; this.cluster = cluster; this.knownK8SPods = knownK8SPods; this.testNamespace = testNamespace; + this.testsType = testsType; + } public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { + var podLabels = new PodLabels(); + podLabels.Add("codex-test-node", "dist-test-" + numberSource.WorkflowNumber); + podLabels.Add("runid", NameUtils.GetRunId()); + podLabels.Add("tests-type", testsType); + podLabels.Add("app", recipeFactory.AppName); + return K8s(controller => { var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); @@ -30,7 +39,7 @@ namespace KubernetesWorkflow var runningPod = controller.BringOnline(recipes, location); return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig)); - }); + }, podLabels); } public void Stop(RunningContainers runningContainers) @@ -147,14 +156,19 @@ namespace KubernetesWorkflow private void K8s(Action action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, new PodLabels()); action(controller); controller.Dispose(); } private T K8s(Func action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); + return K8s(action, new PodLabels()); + } + + private T K8s(Func action, PodLabels podLabels) + { + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels); var result = action(controller); controller.Dispose(); return result; diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index ea1775a..2672f0f 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -10,17 +10,19 @@ namespace KubernetesWorkflow private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; private readonly BaseLog log; + private readonly string testsType; private readonly string testNamespace; - public WorkflowCreator(BaseLog log, Configuration configuration) - : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant()) + public WorkflowCreator(BaseLog log, Configuration configuration, string testsType) + : this(log, configuration, testsType, Guid.NewGuid().ToString().ToLowerInvariant()) { } - public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix) + public WorkflowCreator(BaseLog log, Configuration configuration, string testsType, string testNamespacePostfix) { cluster = new K8sCluster(configuration); this.log = log; + this.testsType = testsType; testNamespace = testNamespacePostfix; } @@ -29,7 +31,7 @@ namespace KubernetesWorkflow var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, testsType); } } } diff --git a/Logging/NameUtils.cs b/Logging/NameUtils.cs index aee7d84..26031b9 100644 --- a/Logging/NameUtils.cs +++ b/Logging/NameUtils.cs @@ -45,7 +45,7 @@ namespace Logging private static string GetEnvVar(string name) { var v = Environment.GetEnvironmentVariable(name); - if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet"; + if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet"; return v; } From 17673e8fa536d977b34a48d602f17eccb97b68cc Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 10 Aug 2023 10:58:18 +0200 Subject: [PATCH 03/10] cleanup some workflow/lifecycle madness --- CodexNetDeployer/CodexNodeStarter.cs | 6 ++--- CodexNetDeployer/Deployer.cs | 25 ++++++------------- CodexNetDownloader/Program.cs | 2 +- ContinuousTests/ContinuousTestRunner.cs | 6 ++--- ContinuousTests/K8sFactory.cs | 20 +++------------ ContinuousTests/NodeRunner.cs | 8 +++--- ContinuousTests/SingleTestRun.cs | 4 +-- DistTestCore/BaseStarter.cs | 7 ++---- DistTestCore/CodexStarter.cs | 6 ++--- DistTestCore/Configuration.cs | 7 ++++-- DistTestCore/GethStarter.cs | 11 ++++---- .../Marketplace/CodexContractsStarter.cs | 6 ++--- .../Marketplace/GethBootstrapNodeStarter.cs | 6 ++--- .../Marketplace/GethCompanionNodeStarter.cs | 6 ++--- DistTestCore/PrometheusStarter.cs | 6 ++--- DistTestCore/TestLifecycle.cs | 14 +++++------ KubernetesWorkflow/StartupWorkflow.cs | 1 - 17 files changed, 56 insertions(+), 85 deletions(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 9478a98..0a3b373 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -8,16 +8,14 @@ namespace CodexNetDeployer public class CodexNodeStarter { private readonly Configuration config; - private readonly WorkflowCreator workflowCreator; private readonly TestLifecycle lifecycle; private readonly GethStartResult gethResult; private string bootstrapSpr = ""; private int validatorsLeft; - public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators) + public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators) { this.config = config; - this.workflowCreator = workflowCreator; this.lifecycle = lifecycle; this.gethResult = gethResult; validatorsLeft = numberOfValidators; @@ -26,7 +24,7 @@ namespace CodexNetDeployer public RunningContainer? Start(int i) { Console.Write($" - {i} = "); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var workflowStartup = new StartupConfig(); workflowStartup.Add(gethResult); workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft)); diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index fc3c425..dda9cd8 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -21,7 +21,7 @@ namespace CodexNetDeployer public CodexDeployment Deploy() { Log("Initializing..."); - var (workflowCreator, lifecycle) = CreateFacilities(); + var lifecycle = CreateTestLifecycle(); Log("Preparing configuration..."); // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. @@ -30,7 +30,7 @@ namespace CodexNetDeployer setup.MetricsEnabled = config.RecordMetrics; Log("Creating Geth instance and deploying contracts..."); - var gethStarter = new GethStarter(lifecycle, workflowCreator); + var gethStarter = new GethStarter(lifecycle); var gethResults = gethStarter.BringOnlineMarketplaceFor(setup); Log("Geth started. Codex contracts deployed."); @@ -44,7 +44,7 @@ namespace CodexNetDeployer Log("Starting Codex nodes..."); // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. - var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value); + var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value); var codexContainers = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { @@ -57,7 +57,7 @@ namespace CodexNetDeployer return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata()); } - private (WorkflowCreator, TestLifecycle) CreateFacilities() + private TestLifecycle CreateTestLifecycle() { var kubeConfig = GetKubeConfig(config.KubeConfigFile); @@ -68,22 +68,11 @@ namespace CodexNetDeployer logDebug: false, dataFilesPath: "notUsed", codexLogLevel: config.CodexLogLevel, - runnerLocation: config.RunnerLocation + runnerLocation: config.RunnerLocation, + k8sNamespacePrefix: config.KubeNamespace ); - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: config.KubeNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: timeset.K8sOperationTimeout(), - retryDelay: timeset.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, - testNamespacePostfix: string.Empty, - testsType: config.TestsTypePodLabel); - - var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); - - return (workflowCreator, lifecycle); + return new TestLifecycle(log, lifecycleConfig, timeset); } private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs index f52a1d2..195e96b 100644 --- a/CodexNetDownloader/Program.cs +++ b/CodexNetDownloader/Program.cs @@ -25,7 +25,7 @@ public class Program if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath); var k8sFactory = new K8sFactory(); - var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); foreach (var container in config.CodexDeployment.CodexContainers) { diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 985707a..35ff486 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -60,8 +60,8 @@ namespace ContinuousTests if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); - var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); - workflowCreator.CreateWorkflow().DeleteTestResources(); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); + lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources(); } private void StartLogDownloader(TaskFactory taskFactory) @@ -71,7 +71,7 @@ namespace ContinuousTests var path = Path.Combine(config.LogPath, "containers"); if (!Directory.Exists(path)) Directory.CreateDirectory(path); - var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken); taskFactory.Run(downloader.Run); diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index fc3a6c0..7695a76 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -1,13 +1,12 @@ using DistTestCore.Codex; using DistTestCore; -using KubernetesWorkflow; using Logging; namespace ContinuousTests { public class K8sFactory { - public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) + public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) { var kubeConfig = GetKubeConfig(kubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration @@ -17,22 +16,11 @@ namespace ContinuousTests logDebug: false, dataFilesPath: dataFilePath, codexLogLevel: CodexLogLevel.Debug, - runnerLocation: runnerLocation + runnerLocation: runnerLocation, + k8sNamespacePrefix: customNamespace ); - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: customNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, - testNamespacePostfix: string.Empty, - testsType: "continuous"); - - var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator); - - return (workflowCreator, lifecycle); + return new TestLifecycle(log, lifecycleConfig, timeSet); } private static string? GetKubeConfig(string kubeConfigFile) diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 02116ee..4bb335e 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -40,8 +40,8 @@ namespace ContinuousTests public void RunNode(CodexAccess bootstrapNode, Action operation, TestToken mintTestTokens) { - var (workflowCreator, lifecycle) = CreateFacilities(); - var flow = workflowCreator.CreateWorkflow(); + var lifecycle = CreateTestLifecycle(); + var flow = lifecycle.WorkflowCreator.CreateWorkflow(); try { @@ -89,9 +89,9 @@ namespace ContinuousTests } } - private (WorkflowCreator, TestLifecycle) CreateFacilities() + private TestLifecycle CreateTestLifecycle() { - return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); + return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); } } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index b065c77..cff325e 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -138,7 +138,7 @@ namespace ContinuousTests private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); - var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); foreach (var container in config.CodexDeployment.CodexContainers) { @@ -221,7 +221,7 @@ namespace ContinuousTests private DistTestCore.Configuration CreateFileManagerConfiguration() { return new DistTestCore.Configuration(null, string.Empty, false, dataFolder, - CodexLogLevel.Error, config.RunnerLocation); + CodexLogLevel.Error, config.RunnerLocation, string.Empty); } } } diff --git a/DistTestCore/BaseStarter.cs b/DistTestCore/BaseStarter.cs index 83a6b8d..4d10643 100644 --- a/DistTestCore/BaseStarter.cs +++ b/DistTestCore/BaseStarter.cs @@ -1,18 +1,15 @@ -using KubernetesWorkflow; -using Logging; +using Logging; namespace DistTestCore { public class BaseStarter { protected readonly TestLifecycle lifecycle; - protected readonly WorkflowCreator workflowCreator; private Stopwatch? stopwatch; - public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + public BaseStarter(TestLifecycle lifecycle) { this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; } protected void LogStart(string msg) diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 690ee94..a1e38b7 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -8,8 +8,8 @@ namespace DistTestCore { public class CodexStarter : BaseStarter { - public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public CodexStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -121,7 +121,7 @@ namespace DistTestCore private StartupWorkflow CreateWorkflow() { - return workflowCreator.CreateWorkflow(); + return lifecycle.WorkflowCreator.CreateWorkflow(); } private void LogSeparator() diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 03514cd..daccad6 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -12,6 +12,7 @@ namespace DistTestCore private readonly string dataFilesPath; private readonly CodexLogLevel codexLogLevel; private readonly TestRunnerLocation runnerLocation; + private readonly string k8sNamespacePrefix; public Configuration() { @@ -21,9 +22,10 @@ namespace DistTestCore dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); + k8sNamespacePrefix = "ct-"; } - public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) + public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix) { this.kubeConfigFile = kubeConfigFile; this.logPath = logPath; @@ -31,12 +33,13 @@ namespace DistTestCore this.dataFilesPath = dataFilesPath; this.codexLogLevel = codexLogLevel; this.runnerLocation = runnerLocation; + this.k8sNamespacePrefix = k8sNamespacePrefix; } public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet) { return new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: "ct-", + k8sNamespacePrefix: k8sNamespacePrefix, kubeConfigFile: kubeConfigFile, operationTimeout: timeSet.K8sOperationTimeout(), retryDelay: timeSet.WaitForK8sServiceDelay() diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 3b9a9b1..578cc30 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -1,5 +1,4 @@ using DistTestCore.Marketplace; -using KubernetesWorkflow; namespace DistTestCore { @@ -8,13 +7,13 @@ namespace DistTestCore private readonly MarketplaceNetworkCache marketplaceNetworkCache; private readonly GethCompanionNodeStarter companionNodeStarter; - public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethStarter(TestLifecycle lifecycle) + : base(lifecycle) { marketplaceNetworkCache = new MarketplaceNetworkCache( - new GethBootstrapNodeStarter(lifecycle, workflowCreator), - new CodexContractsStarter(lifecycle, workflowCreator)); - companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); + new GethBootstrapNodeStarter(lifecycle), + new CodexContractsStarter(lifecycle)); + companionNodeStarter = new GethCompanionNodeStarter(lifecycle); } public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup) diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index c41df53..a1db57f 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace public class CodexContractsStarter : BaseStarter { - public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public CodexContractsStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace { LogStart("Deploying Codex Marketplace..."); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]); var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 56296ff..d1ebb54 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeStarter : BaseStarter { - public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethBootstrapNodeStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace LogStart("Starting Geth bootstrap node..."); var startupConfig = CreateBootstrapStartupConfig(); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); var bootstrapContainer = containers.Containers[0]; diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index 2008e9c..6759e7b 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace { private int companionAccountIndex = 0; - public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethCompanionNodeStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config)); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 6ef2e57..1804b77 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -7,8 +7,8 @@ namespace DistTestCore { public class PrometheusStarter : BaseStarter { - public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public PrometheusStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -18,7 +18,7 @@ namespace DistTestCore var startupConfig = new StartupConfig(); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers()))); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 4c674cf..26c0a5a 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -11,20 +11,17 @@ namespace DistTestCore private readonly DateTime testStart; public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) - : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests")) - { - } - - public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) { Log = log; Configuration = configuration; TimeSet = timeSet; + WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests"); + FileManager = new FileManager(Log, configuration); - CodexStarter = new CodexStarter(this, workflowCreator); - PrometheusStarter = new PrometheusStarter(this, workflowCreator); - GethStarter = new GethStarter(this, workflowCreator); + CodexStarter = new CodexStarter(this); + PrometheusStarter = new PrometheusStarter(this); + GethStarter = new GethStarter(this); testStart = DateTime.UtcNow; CodexVersion = null; @@ -34,6 +31,7 @@ namespace DistTestCore public BaseLog Log { get; } public Configuration Configuration { get; } public ITimeSet TimeSet { get; } + public WorkflowCreator WorkflowCreator { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index a4b9df0..67f757f 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -27,7 +27,6 @@ namespace KubernetesWorkflow public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { var podLabels = new PodLabels(); - podLabels.Add("codex-test-node", "dist-test-" + numberSource.WorkflowNumber); podLabels.Add("runid", NameUtils.GetRunId()); podLabels.Add("tests-type", testsType); podLabels.Add("app", recipeFactory.AppName); From 01ec6f7d8b7f8d779435b888e3910ae5c0615c0b Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 10 Aug 2023 11:25:22 +0200 Subject: [PATCH 04/10] sets up additional pod labels --- CodexNetDeployer/Deployer.cs | 2 +- ContinuousTests/K8sFactory.cs | 2 +- DistTestCore/DistTest.cs | 15 ++++----------- DistTestCore/TestLifecycle.cs | 13 +++++++++++-- KubernetesWorkflow/PodLabels.cs | 22 ++++++++++++++++++++-- KubernetesWorkflow/StartupWorkflow.cs | 21 ++++++--------------- KubernetesWorkflow/WorkflowCreator.cs | 15 +++++---------- 7 files changed, 48 insertions(+), 42 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index dda9cd8..32e08c7 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -72,7 +72,7 @@ namespace CodexNetDeployer k8sNamespacePrefix: config.KubeNamespace ); - return new TestLifecycle(log, lifecycleConfig, timeset); + return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel); } private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index 7695a76..5627941 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -20,7 +20,7 @@ namespace ContinuousTests k8sNamespacePrefix: customNamespace ); - return new TestLifecycle(log, lifecycleConfig, timeSet); + return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests"); } private static string? GetKubeConfig(string kubeConfigFile) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index e05c566..48ae884 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -13,6 +13,7 @@ namespace DistTestCore [Parallelizable(ParallelScope.All)] public abstract class DistTest { + private const string TestsType = "dist-tests"; private readonly Configuration configuration = new Configuration(); private readonly Assembly[] testAssemblies; private readonly FixtureLog fixtureLog; @@ -52,7 +53,7 @@ namespace DistTestCore { Stopwatch.Measure(fixtureLog, "Global setup", () => { - var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), "dist-tests"); + var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, "null")); wc.CreateWorkflow().DeleteAllResources(); }); } @@ -195,7 +196,7 @@ namespace DistTestCore { lock (lifecycleLock) { - lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet())); + lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType)); } }); } @@ -206,7 +207,7 @@ namespace DistTestCore var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})"); - statusLog.ConcludeTest(testResult, testDuration, GetCodexId(lifecycle)); + statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetCodexId()); Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => { lifecycle.Log.EndTest(); @@ -216,14 +217,6 @@ namespace DistTestCore }); } - private static string GetCodexId(TestLifecycle lifecycle) - { - var v = lifecycle.CodexVersion; - if (v == null) return new CodexContainerRecipe().Image; - if (v.version != "untagged build") return v.version; - return v.revision; - } - private ITimeSet GetTimeSet() { if (ShouldUseLongTimeouts()) return new LongTimeSet(); diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 26c0a5a..cc98ec5 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -10,13 +10,14 @@ namespace DistTestCore { private readonly DateTime testStart; - public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) + public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType) { Log = log; Configuration = configuration; TimeSet = timeSet; - WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests"); + var podLabels = new PodLabels(testsType, GetCodexId()); + WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels); FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this); @@ -66,5 +67,13 @@ namespace DistTestCore { if (CodexVersion == null) CodexVersion = version; } + + public string GetCodexId() + { + var v = CodexVersion; + if (v == null) return new CodexContainerRecipe().Image; + if (v.version != "untagged build") return v.version; + return v.revision; + } } } diff --git a/KubernetesWorkflow/PodLabels.cs b/KubernetesWorkflow/PodLabels.cs index 8c444bf..e1097d5 100644 --- a/KubernetesWorkflow/PodLabels.cs +++ b/KubernetesWorkflow/PodLabels.cs @@ -1,10 +1,28 @@ -namespace KubernetesWorkflow +using Logging; + +namespace KubernetesWorkflow { public class PodLabels { private readonly Dictionary labels = new Dictionary(); - public void Add(string key, string value) + public PodLabels(string testsType, string codexId) + { + Add("tests-type", testsType); + Add("runid", NameUtils.GetRunId()); + Add("testid", NameUtils.GetTestId()); + Add("category", NameUtils.GetCategoryName()); + Add("codexid", codexId); + Add("fixturename", NameUtils.GetRawFixtureName()); + Add("testname", NameUtils.GetTestMethodName()); + } + + public void AddAppName(string appName) + { + Add("app", appName); + } + + private void Add(string key, string value) { labels.Add(key, value.ToLowerInvariant()); } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 67f757f..185980b 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -10,26 +10,22 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownK8SPods; private readonly string testNamespace; - private readonly string testsType; + private readonly PodLabels podLabels; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, string testsType) + internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, PodLabels podLabels) { this.log = log; this.numberSource = numberSource; this.cluster = cluster; this.knownK8SPods = knownK8SPods; this.testNamespace = testNamespace; - this.testsType = testsType; - + this.podLabels = podLabels; } public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { - var podLabels = new PodLabels(); - podLabels.Add("runid", NameUtils.GetRunId()); - podLabels.Add("tests-type", testsType); - podLabels.Add("app", recipeFactory.AppName); + podLabels.AddAppName(recipeFactory.AppName); return K8s(controller => { @@ -38,7 +34,7 @@ namespace KubernetesWorkflow var runningPod = controller.BringOnline(recipes, location); return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig)); - }, podLabels); + }); } public void Stop(RunningContainers runningContainers) @@ -155,17 +151,12 @@ namespace KubernetesWorkflow private void K8s(Action action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, new PodLabels()); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels); action(controller); controller.Dispose(); } private T K8s(Func action) - { - return K8s(action, new PodLabels()); - } - - private T K8s(Func action, PodLabels podLabels) { var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels); var result = action(controller); diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 2672f0f..afc515f 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -10,20 +10,15 @@ namespace KubernetesWorkflow private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; private readonly BaseLog log; - private readonly string testsType; + private readonly PodLabels podLabels; private readonly string testNamespace; - public WorkflowCreator(BaseLog log, Configuration configuration, string testsType) - : this(log, configuration, testsType, Guid.NewGuid().ToString().ToLowerInvariant()) - { - } - - public WorkflowCreator(BaseLog log, Configuration configuration, string testsType, string testNamespacePostfix) + public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels) { cluster = new K8sCluster(configuration); this.log = log; - this.testsType = testsType; - testNamespace = testNamespacePostfix; + this.podLabels = podLabels; + testNamespace = Guid.NewGuid().ToString().ToLowerInvariant(); } public StartupWorkflow CreateWorkflow() @@ -31,7 +26,7 @@ namespace KubernetesWorkflow var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, testsType); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, podLabels); } } } From 5a861df96857ebc129b413a7fa16b8c5590c1c5a Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 10 Aug 2023 11:47:34 +0200 Subject: [PATCH 05/10] local tests passed --- KubernetesWorkflow/PodLabels.cs | 18 +++++++++++++++--- KubernetesWorkflow/StartupWorkflow.cs | 12 ++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/KubernetesWorkflow/PodLabels.cs b/KubernetesWorkflow/PodLabels.cs index e1097d5..e73a348 100644 --- a/KubernetesWorkflow/PodLabels.cs +++ b/KubernetesWorkflow/PodLabels.cs @@ -6,6 +6,11 @@ namespace KubernetesWorkflow { private readonly Dictionary labels = new Dictionary(); + private PodLabels(PodLabels source) + { + labels = source.labels.ToDictionary(p => p.Key, p => p.Value); + } + public PodLabels(string testsType, string codexId) { Add("tests-type", testsType); @@ -17,14 +22,21 @@ namespace KubernetesWorkflow Add("testname", NameUtils.GetTestMethodName()); } - public void AddAppName(string appName) + public PodLabels GetLabelsForAppName(string appName) { - Add("app", appName); + var pl = new PodLabels(this); + pl.Add("app", appName); + return pl; } private void Add(string key, string value) { - labels.Add(key, value.ToLowerInvariant()); + labels.Add(key, + value.ToLowerInvariant() + .Replace(":","-") + .Replace("/", "-") + .Replace("\\", "-") + ); } internal Dictionary GetLabels() diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 185980b..c51b991 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -25,7 +25,7 @@ namespace KubernetesWorkflow public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { - podLabels.AddAppName(recipeFactory.AppName); + var pl = podLabels.GetLabelsForAppName(recipeFactory.AppName); return K8s(controller => { @@ -34,7 +34,7 @@ namespace KubernetesWorkflow var runningPod = controller.BringOnline(recipes, location); return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig)); - }); + }, pl); } public void Stop(RunningContainers runningContainers) @@ -163,6 +163,14 @@ namespace KubernetesWorkflow controller.Dispose(); return result; } + + private T K8s(Func action, PodLabels labels) + { + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, labels); + var result = action(controller); + controller.Dispose(); + return result; + } } public interface ILogHandler From 4995665fe5b57951c7d4b2ec6d94287c796de8f6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 10 Aug 2023 11:53:49 +0200 Subject: [PATCH 06/10] deals with adhoc test context categories --- Logging/NameUtils.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Logging/NameUtils.cs b/Logging/NameUtils.cs index 26031b9..2961735 100644 --- a/Logging/NameUtils.cs +++ b/Logging/NameUtils.cs @@ -29,6 +29,7 @@ namespace Logging public static string GetCategoryName() { var test = TestContext.CurrentContext.Test; + if (test.ClassName!.Contains("AdhocContext")) return "-"; return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.')); } From 9c1b469f0d071ee7ec3f3432dd53255aa35ecf09 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 10 Aug 2023 11:56:29 +0200 Subject: [PATCH 07/10] same for fixture name --- Logging/NameUtils.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Logging/NameUtils.cs b/Logging/NameUtils.cs index 2961735..9d63398 100644 --- a/Logging/NameUtils.cs +++ b/Logging/NameUtils.cs @@ -22,6 +22,7 @@ namespace Logging public static string GetRawFixtureName() { var test = TestContext.CurrentContext.Test; + if (test.ClassName!.Contains("AdhocContext")) return "-"; var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); return className.Replace('.', '-'); } From 683cc882be1147d6da4d25bae35dceb2eec1bca3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 10 Aug 2023 11:59:57 +0200 Subject: [PATCH 08/10] dash is disallowed --- Logging/NameUtils.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Logging/NameUtils.cs b/Logging/NameUtils.cs index 9d63398..2ca47e2 100644 --- a/Logging/NameUtils.cs +++ b/Logging/NameUtils.cs @@ -22,7 +22,7 @@ namespace Logging public static string GetRawFixtureName() { var test = TestContext.CurrentContext.Test; - if (test.ClassName!.Contains("AdhocContext")) return "-"; + if (test.ClassName!.Contains("AdhocContext")) return "none"; var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); return className.Replace('.', '-'); } @@ -30,7 +30,7 @@ namespace Logging public static string GetCategoryName() { var test = TestContext.CurrentContext.Test; - if (test.ClassName!.Contains("AdhocContext")) return "-"; + if (test.ClassName!.Contains("AdhocContext")) return "none"; return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.')); } From c7e5ddb0cb6ec3d638e09e8246bbcdb5abbbc0c5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 10 Aug 2023 13:58:50 +0200 Subject: [PATCH 09/10] additional application ids, fixes deployment namespace issue --- CodexNetDeployer/Deployer.cs | 2 +- ContinuousTests/K8sFactory.cs | 2 +- DistTestCore/DistTest.cs | 8 +++++--- DistTestCore/TestLifecycle.cs | 20 ++++++++++++++++---- KubernetesWorkflow/PodLabels.cs | 9 +++++++-- KubernetesWorkflow/WorkflowCreator.cs | 4 ++-- Logging/ApplicationIds.cs | 18 ++++++++++++++++++ Logging/StatusLog.cs | 10 ++++++++-- 8 files changed, 58 insertions(+), 15 deletions(-) create mode 100644 Logging/ApplicationIds.cs diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 32e08c7..a5e174e 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -72,7 +72,7 @@ namespace CodexNetDeployer k8sNamespacePrefix: config.KubeNamespace ); - return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel); + return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); } private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index 5627941..ff9b7e9 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -20,7 +20,7 @@ namespace ContinuousTests k8sNamespacePrefix: customNamespace ); - return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests"); + return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty); } private static string? GetKubeConfig(string kubeConfigFile) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 48ae884..a38377d 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -53,7 +53,7 @@ namespace DistTestCore { Stopwatch.Measure(fixtureLog, "Global setup", () => { - var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, "null")); + var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty); wc.CreateWorkflow().DeleteAllResources(); }); } @@ -196,7 +196,9 @@ namespace DistTestCore { lock (lifecycleLock) { - lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType)); + var testNamespace = Guid.NewGuid().ToString(); + var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace); + lifecycles.Add(testName, lifecycle); } }); } @@ -207,7 +209,7 @@ namespace DistTestCore var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})"); - statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetCodexId()); + statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds()); Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => { lifecycle.Log.EndTest(); diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index cc98ec5..eede32d 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,5 +1,7 @@ using DistTestCore.Codex; using DistTestCore.Logs; +using DistTestCore.Marketplace; +using DistTestCore.Metrics; using KubernetesWorkflow; using Logging; using Utils; @@ -10,14 +12,14 @@ namespace DistTestCore { private readonly DateTime testStart; - public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType) + public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace) { Log = log; Configuration = configuration; TimeSet = timeSet; - var podLabels = new PodLabels(testsType, GetCodexId()); - WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels); + var podLabels = new PodLabels(testsType, GetApplicationIds()); + WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace); FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this); @@ -68,7 +70,17 @@ namespace DistTestCore if (CodexVersion == null) CodexVersion = version; } - public string GetCodexId() + public ApplicationIds GetApplicationIds() + { + return new ApplicationIds( + codexId: GetCodexId(), + gethId: new GethContainerRecipe().Image, + prometheusId: new PrometheusContainerRecipe().Image, + codexContractsId: new CodexContractsContainerRecipe().Image + ); + } + + private string GetCodexId() { var v = CodexVersion; if (v == null) return new CodexContainerRecipe().Image; diff --git a/KubernetesWorkflow/PodLabels.cs b/KubernetesWorkflow/PodLabels.cs index e73a348..60fc332 100644 --- a/KubernetesWorkflow/PodLabels.cs +++ b/KubernetesWorkflow/PodLabels.cs @@ -11,15 +11,20 @@ namespace KubernetesWorkflow labels = source.labels.ToDictionary(p => p.Key, p => p.Value); } - public PodLabels(string testsType, string codexId) + public PodLabels(string testsType, ApplicationIds applicationIds) { Add("tests-type", testsType); Add("runid", NameUtils.GetRunId()); Add("testid", NameUtils.GetTestId()); Add("category", NameUtils.GetCategoryName()); - Add("codexid", codexId); Add("fixturename", NameUtils.GetRawFixtureName()); Add("testname", NameUtils.GetTestMethodName()); + + if (applicationIds == null) return; + Add("codexid", applicationIds.CodexId); + Add("gethid", applicationIds.GethId); + Add("prometheusid", applicationIds.PrometheusId); + Add("codexcontractsid", applicationIds.CodexContractsId); } public PodLabels GetLabelsForAppName(string appName) diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index afc515f..4e1c346 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -13,12 +13,12 @@ namespace KubernetesWorkflow private readonly PodLabels podLabels; private readonly string testNamespace; - public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels) + public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels, string testNamespace) { cluster = new K8sCluster(configuration); this.log = log; this.podLabels = podLabels; - testNamespace = Guid.NewGuid().ToString().ToLowerInvariant(); + this.testNamespace = testNamespace.ToLowerInvariant(); } public StartupWorkflow CreateWorkflow() diff --git a/Logging/ApplicationIds.cs b/Logging/ApplicationIds.cs new file mode 100644 index 0000000..d52dc10 --- /dev/null +++ b/Logging/ApplicationIds.cs @@ -0,0 +1,18 @@ +namespace Logging +{ + public class ApplicationIds + { + public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId) + { + CodexId = codexId; + GethId = gethId; + PrometheusId = prometheusId; + CodexContractsId = codexContractsId; + } + + public string CodexId { get; } + public string GethId { get; } + public string PrometheusId { get; } + public string CodexContractsId { get; } + } +} diff --git a/Logging/StatusLog.cs b/Logging/StatusLog.cs index f0a366b..96a8cc2 100644 --- a/Logging/StatusLog.cs +++ b/Logging/StatusLog.cs @@ -14,7 +14,7 @@ namespace Logging fixtureName = NameUtils.GetRawFixtureName(); } - public void ConcludeTest(string resultStatus, string testDuration, string codexId) + public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds) { Write(new StatusLogJson { @@ -22,7 +22,10 @@ namespace Logging runid = NameUtils.GetRunId(), status = resultStatus, testid = NameUtils.GetTestId(), - codexid = codexId, + codexid = applicationIds.CodexId, + gethid = applicationIds.GethId, + prometheusid = applicationIds.PrometheusId, + codexcontractsid = applicationIds.CodexContractsId, category = NameUtils.GetCategoryName(), fixturename = fixtureName, testname = NameUtils.GetTestMethodName(), @@ -53,6 +56,9 @@ namespace Logging public string status { get; set; } = string.Empty; public string testid { get; set; } = string.Empty; public string codexid { get; set; } = string.Empty; + public string gethid { get; set; } = string.Empty; + public string prometheusid { get; set; } = string.Empty; + public string codexcontractsid { get; set; } = string.Empty; public string category { get; set; } = string.Empty; public string fixturename { get; set; } = string.Empty; public string testname { get; set; } = string.Empty; From 3de8e29198c310571e51411dceda28307e6a1283 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 10 Aug 2023 14:25:32 +0200 Subject: [PATCH 10/10] turns on metrics by default for deployment --- CodexNetDeployer/deploy-continuous-testnet.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh index 5337922..338b8c1 100644 --- a/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -9,5 +9,5 @@ dotnet run \ --min-price=1024 \ --max-collateral=1024 \ --max-duration=3600000 \ - --block-ttl=120 \ - -y + --block-ttl=300 \ + --record-metrics=true