From 0259c30bb509b0d81c3e90ac1bd63393ed380a40 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 7 Aug 2023 15:51:44 +0200 Subject: [PATCH] Wires up pod labels --- CodexNetDeployer/Configuration.cs | 4 ++++ CodexNetDeployer/Deployer.cs | 5 ++++- ContinuousTests/K8sFactory.cs | 5 ++++- DistTestCore/Codex/CodexContainerRecipe.cs | 1 + DistTestCore/DistTest.cs | 2 +- .../CodexContractsContainerRecipe.cs | 8 ++----- .../Marketplace/GethContainerRecipe.cs | 8 ++----- .../Metrics/PrometheusContainerRecipe.cs | 8 ++----- DistTestCore/TestLifecycle.cs | 2 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 1 + KubernetesWorkflow/K8sController.cs | 12 ++++------ KubernetesWorkflow/PodLabels.cs | 17 ++++++++++++++ KubernetesWorkflow/StartupWorkflow.cs | 22 +++++++++++++++---- KubernetesWorkflow/WorkflowCreator.cs | 10 +++++---- Logging/NameUtils.cs | 2 +- 15 files changed, 68 insertions(+), 39 deletions(-) create mode 100644 KubernetesWorkflow/PodLabels.cs diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index b030618..955084a 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -46,6 +46,10 @@ namespace CodexNetDeployer [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] public bool RecordMetrics { get; set; } = false; + + [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " + + "set this option to override the label value.")] + public string TestsTypePodLabel { get; set; } = "continuous"; public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 4bfa25d..fc3c425 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -77,7 +77,10 @@ namespace CodexNetDeployer operationTimeout: timeset.K8sOperationTimeout(), retryDelay: timeset.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); + var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, + testNamespacePostfix: string.Empty, + testsType: config.TestsTypePodLabel); + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); return (workflowCreator, lifecycle); diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index 0704e9f..fc3a6c0 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -26,7 +26,10 @@ namespace ContinuousTests operationTimeout: timeSet.K8sOperationTimeout(), retryDelay: timeSet.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); + var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, + testNamespacePostfix: string.Empty, + testsType: "continuous"); + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator); return (workflowCreator, lifecycle); diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index ea6f187..c600c01 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -14,6 +14,7 @@ namespace DistTestCore.Codex public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); + public override string AppName => "codex"; public override string Image { get; } public CodexContainerRecipe() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 6768618..e05c566 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -52,7 +52,7 @@ namespace DistTestCore { Stopwatch.Measure(fixtureLog, "Global setup", () => { - var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet())); + var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), "dist-tests"); wc.CreateWorkflow().DeleteAllResources(); }); } diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index d4bc5d1..92a6c81 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -7,12 +7,8 @@ namespace DistTestCore.Marketplace public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; - public override string Image { get; } - - public CodexContractsContainerRecipe() - { - Image = "codexstorage/dist-tests-codex-contracts-eth:sha-9a83699"; - } + public override string AppName => "codex-contracts"; + public override string Image => "codexstorage/dist-tests-codex-contracts-eth:sha-9a83699"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 6dc62e9..4ca7895 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -10,12 +10,8 @@ namespace DistTestCore.Marketplace public const string DiscoveryPortTag = "disc_port"; public const string AccountsFilename = "accounts.csv"; - public override string Image { get; } - - public GethContainerRecipe() - { - Image = "codexstorage/dist-tests-geth:sha-b788a2d"; - } + public override string AppName => "geth"; + public override string Image => "codexstorage/dist-tests-geth:sha-b788a2d"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs index b702222..e51fdc5 100644 --- a/DistTestCore/Metrics/PrometheusContainerRecipe.cs +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -4,12 +4,8 @@ namespace DistTestCore.Metrics { public class PrometheusContainerRecipe : ContainerRecipeFactory { - public override string Image { get; } - - public PrometheusContainerRecipe() - { - Image = "codexstorage/dist-tests-prometheus:sha-f97d7fd"; - } + public override string AppName => "prometheus"; + public override string Image => "codexstorage/dist-tests-prometheus:sha-f97d7fd"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index f33b238..4c674cf 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -11,7 +11,7 @@ namespace DistTestCore private readonly DateTime testStart; public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) - : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet))) + : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), "dist-tests")) { } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 340ed2d..470bd42 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -27,6 +27,7 @@ return recipe; } + public abstract string AppName { get; } public abstract string Image { get; } protected int ContainerNumber { get; private set; } = 0; protected int Index { get; private set; } = 0; diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index e8dc393..5569b07 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -11,14 +11,16 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; + private readonly PodLabels podLabels; private readonly K8sClient client; - public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace) + public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace, PodLabels podLabels) { this.log = log; this.cluster = cluster; this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; + this.podLabels = podLabels; client = new K8sClient(cluster.GetK8sClientConfig()); K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace; @@ -362,13 +364,7 @@ namespace KubernetesWorkflow private IDictionary GetSelector() { - return new Dictionary - { - { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } - // tests-type=dist-tests - // app=codex - // runid=20230721-085043 - }; + return podLabels.GetLabels(); } private IDictionary GetRunnerNamespaceSelector() diff --git a/KubernetesWorkflow/PodLabels.cs b/KubernetesWorkflow/PodLabels.cs new file mode 100644 index 0000000..8c444bf --- /dev/null +++ b/KubernetesWorkflow/PodLabels.cs @@ -0,0 +1,17 @@ +namespace KubernetesWorkflow +{ + public class PodLabels + { + private readonly Dictionary labels = new Dictionary(); + + public void Add(string key, string value) + { + labels.Add(key, value.ToLowerInvariant()); + } + + internal Dictionary GetLabels() + { + return labels; + } + } +} diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index c7e3bfb..a4b9df0 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -10,19 +10,28 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownK8SPods; private readonly string testNamespace; + private readonly string testsType; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace) + internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, string testsType) { this.log = log; this.numberSource = numberSource; this.cluster = cluster; this.knownK8SPods = knownK8SPods; this.testNamespace = testNamespace; + this.testsType = testsType; + } public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { + var podLabels = new PodLabels(); + podLabels.Add("codex-test-node", "dist-test-" + numberSource.WorkflowNumber); + podLabels.Add("runid", NameUtils.GetRunId()); + podLabels.Add("tests-type", testsType); + podLabels.Add("app", recipeFactory.AppName); + return K8s(controller => { var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); @@ -30,7 +39,7 @@ namespace KubernetesWorkflow var runningPod = controller.BringOnline(recipes, location); return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig)); - }); + }, podLabels); } public void Stop(RunningContainers runningContainers) @@ -147,14 +156,19 @@ namespace KubernetesWorkflow private void K8s(Action action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, new PodLabels()); action(controller); controller.Dispose(); } private T K8s(Func action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); + return K8s(action, new PodLabels()); + } + + private T K8s(Func action, PodLabels podLabels) + { + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels); var result = action(controller); controller.Dispose(); return result; diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index ea1775a..2672f0f 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -10,17 +10,19 @@ namespace KubernetesWorkflow private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; private readonly BaseLog log; + private readonly string testsType; private readonly string testNamespace; - public WorkflowCreator(BaseLog log, Configuration configuration) - : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant()) + public WorkflowCreator(BaseLog log, Configuration configuration, string testsType) + : this(log, configuration, testsType, Guid.NewGuid().ToString().ToLowerInvariant()) { } - public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix) + public WorkflowCreator(BaseLog log, Configuration configuration, string testsType, string testNamespacePostfix) { cluster = new K8sCluster(configuration); this.log = log; + this.testsType = testsType; testNamespace = testNamespacePostfix; } @@ -29,7 +31,7 @@ namespace KubernetesWorkflow var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, testsType); } } } diff --git a/Logging/NameUtils.cs b/Logging/NameUtils.cs index aee7d84..26031b9 100644 --- a/Logging/NameUtils.cs +++ b/Logging/NameUtils.cs @@ -45,7 +45,7 @@ namespace Logging private static string GetEnvVar(string name) { var v = Environment.GetEnvironmentVariable(name); - if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet"; + if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet"; return v; }