Feature/add annotations for prometheus (#49)

* Add Prometheus annotations for Codex Pods (#48)

* Remove unnecessary condition (#48)

* Update Annotations to be handled via ContainerRecipe (#48)

* Wires up metrics port to codex pod annotation

* Cleans up handling of pod labels

* Fix annotations names (#48)

---------

Co-authored-by: benbierens <thatbenbierens@gmail.com>

CodexNetDeployer/Deployer.cs

@@ -74,7 +74,10 @@ namespace CodexNetDeployer
                 k8sNamespacePrefix: config.KubeNamespace
             );
 
-            return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
+            var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeset, string.Empty);
+            DefaultContainerRecipe.TestsType = config.TestsTypePodLabel;
+            DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
+            return lifecycle;
         }
 
         private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)

ContinuousTests/K8sFactory.cs

@@ -19,7 +19,10 @@ namespace ContinuousTests
                 k8sNamespacePrefix: customNamespace
             );
 
-            return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, string.Empty);
+            DefaultContainerRecipe.TestsType = "continuous-tests";
+            DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
+            return lifecycle;
         }
 
         private static string? GetKubeConfig(string kubeConfigFile)

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -3,14 +3,14 @@ using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
 {
-    public class CodexContainerRecipe : ContainerRecipeFactory
+    public class CodexContainerRecipe : DefaultContainerRecipe
     {
         private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
 
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
 
-        // Used by tests for time-constraint assersions.
+        // Used by tests for time-constraint assertions.
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
         public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 
@@ -22,7 +22,7 @@ namespace DistTestCore.Codex
             Image = GetDockerImage();
         }
 
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<CodexStartupConfig>();
 
@@ -61,9 +61,12 @@ namespace DistTestCore.Codex
             }
             if (config.MetricsMode != Metrics.MetricsMode.None)
             {
+                var metricsPort = AddInternalPort(MetricsPortTag);
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
-                AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
+                AddEnvVar("CODEX_METRICS_PORT", metricsPort);
+                AddPodAnnotation("prometheus.io/scrape", "true");
+                AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
             }
 
             if (config.MarketplaceConfig != null)

DistTestCore/DefaultContainerRecipe.cs

@@ -0,0 +1,40 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace DistTestCore
+{
+    public abstract class DefaultContainerRecipe : ContainerRecipeFactory
+    {
+        public static string TestsType { get; set; } = "NotSet";
+        public static ApplicationIds? ApplicationIds { get; set; } = null;
+
+        protected abstract void InitializeRecipe(StartupConfig config);
+
+        protected override void Initialize(StartupConfig config)
+        {
+            Add("tests-type", TestsType);
+            Add("runid", NameUtils.GetRunId());
+            Add("testid", NameUtils.GetTestId());
+            Add("category", NameUtils.GetCategoryName());
+            Add("fixturename", NameUtils.GetRawFixtureName());
+            Add("testname", NameUtils.GetTestMethodName());
+
+            if (ApplicationIds != null)
+            {
+                Add("codexid", ApplicationIds.CodexId);
+                Add("gethid", ApplicationIds.GethId);
+                Add("prometheusid", ApplicationIds.PrometheusId);
+                Add("codexcontractsid", ApplicationIds.CodexContractsId);
+                Add("grafanaid", ApplicationIds.GrafanaId);
+            }
+            Add("app", AppName);
+
+            InitializeRecipe(config);
+        }
+
+        private void Add(string name, string value)
+        {
+            AddPodLabel(name, value);
+        }
+    }
+}

DistTestCore/DistTest.cs

@@ -47,7 +47,7 @@ namespace DistTestCore
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty);
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), string.Empty);
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -206,8 +206,10 @@ namespace DistTestCore
                 lock (lifecycleLock)
                 {
                     var testNamespace = Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace);
+                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
                     lifecycles.Add(testName, lifecycle);
+                    DefaultContainerRecipe.TestsType = TestsType;
+                    DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
                 }
             });
         }

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -2,7 +2,7 @@
 
 namespace DistTestCore.Marketplace
 {
-    public class CodexContractsContainerRecipe : ContainerRecipeFactory
+    public class CodexContractsContainerRecipe : DefaultContainerRecipe
     {
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
@@ -10,7 +10,7 @@ namespace DistTestCore.Marketplace
         public override string AppName => "codex-contracts";
         public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
 
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -2,7 +2,7 @@
 
 namespace DistTestCore.Marketplace
 {
-    public class GethContainerRecipe : ContainerRecipeFactory
+    public class GethContainerRecipe : DefaultContainerRecipe
     {
         private const string defaultArgs = "--ipcdisable --syncmode full";
 
@@ -13,7 +13,7 @@ namespace DistTestCore.Marketplace
         public override string AppName => "geth";
         public override string Image => "codexstorage/dist-tests-geth:latest";
 
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<GethStartupConfig>();
 

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -2,7 +2,7 @@
 
 namespace DistTestCore.Metrics
 {
-    public class GrafanaContainerRecipe : ContainerRecipeFactory
+    public class GrafanaContainerRecipe : DefaultContainerRecipe
     {
         public override string AppName => "grafana";
         public override string Image => "grafana/grafana-oss:10.0.3";
@@ -10,7 +10,7 @@ namespace DistTestCore.Metrics
         public const string DefaultAdminUser = "adminium";
         public const string DefaultAdminPassword = "passwordium";
 
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             AddExposedPort(3000);
 

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -2,12 +2,12 @@
 
 namespace DistTestCore.Metrics
 {
-    public class PrometheusContainerRecipe : ContainerRecipeFactory
+    public class PrometheusContainerRecipe : DefaultContainerRecipe
     {
         public override string AppName => "prometheus";
         public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 

DistTestCore/TestLifecycle.cs

@@ -12,14 +12,13 @@ namespace DistTestCore
     {
         private readonly DateTime testStart;
 
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
         {
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
 
-            var podLabels = new PodLabels(testsType, GetApplicationIds());
+            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace);
-            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace);
 
             FileManager = new FileManager(Log, configuration);
             CodexStarter = new CodexStarter(this);

KubernetesWorkflow/CommandRunner.cs

@@ -1,5 +1,4 @@
-using k8s;
+using Utils;
-using Utils;
 
 namespace KubernetesWorkflow
 {

KubernetesWorkflow/ContainerRecipe.cs

@@ -2,13 +2,15 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, object[] additionals)
+        public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, object[] additionals)
         {
             Number = number;
             Image = image;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
+            PodLabels = podLabels;
+            PodAnnotations = podAnnotations;
             Additionals = additionals;
         }
 
@@ -18,6 +20,8 @@
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
         public EnvVar[] EnvVars { get; }
+        public PodLabels PodLabels { get; }
+        public PodAnnotations PodAnnotations { get; }
         public object[] Additionals { get; }
 
         public Port GetPortByTag(string tag)

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -5,6 +5,8 @@
         private readonly List<Port> exposedPorts = new List<Port>();
         private readonly List<Port> internalPorts = new List<Port>();
         private readonly List<EnvVar> envVars = new List<EnvVar>();
+        private readonly PodLabels podLabels = new PodLabels();
+        private readonly PodAnnotations podAnnotations = new PodAnnotations();
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
 
@@ -16,11 +18,19 @@
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), additionals.ToArray());
+            var recipe = new ContainerRecipe(containerNumber, Image,
+                exposedPorts.ToArray(),
+                internalPorts.ToArray(), 
+                envVars.ToArray(), 
+                podLabels.Clone(),
+                podAnnotations.Clone(),
+                additionals.ToArray());
 
             exposedPorts.Clear();
             internalPorts.Clear();
             envVars.Clear();
+            podLabels.Clear();
+            podAnnotations.Clear();
             additionals.Clear();
             this.factory = null!;
 
@@ -74,6 +84,16 @@
             envVars.Add(factory.CreateEnvVar(name, value.Number));
         }
 
+        protected void AddPodLabel(string name, string value)
+        {
+            podLabels.Add(name, value);
+        }
+
+        protected void AddPodAnnotation(string name, string value)
+        {
+            podAnnotations.Add(name, value);
+        }
+
         protected void Additional(object userData)
         {
             additionals.Add(userData);

KubernetesWorkflow/K8sController.cs

@@ -11,16 +11,14 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
-        private readonly PodLabels podLabels;
         private readonly K8sClient client;
 
-        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace, PodLabels podLabels)
+        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
         {
             this.log = log;
             this.cluster = cluster;
             this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
-            this.podLabels = podLabels;
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
@@ -274,7 +272,7 @@ namespace KubernetesWorkflow
                                     {
                                         IpBlock = new V1IPBlock
                                         {
-                                          Cidr = "0.0.0.0/0"
+                                            Cidr = "0.0.0.0/0"
                                         }
                                     }
                                 },
@@ -316,19 +314,20 @@ namespace KubernetesWorkflow
             var deploymentSpec = new V1Deployment
             {
                 ApiVersion = "apps/v1",
-                Metadata = CreateDeploymentMetadata(),
+                Metadata = CreateDeploymentMetadata(containerRecipes),
                 Spec = new V1DeploymentSpec
                 {
                     Replicas = 1,
                     Selector = new V1LabelSelector
                     {
-                        MatchLabels = GetSelector()
+                        MatchLabels = GetSelector(containerRecipes)
                     },
                     Template = new V1PodTemplateSpec
                     {
                         Metadata = new V1ObjectMeta
                         {
-                            Labels = GetSelector()
+                            Labels = GetSelector(containerRecipes),
+                            Annotations = GetAnnotations(containerRecipes)
                         },
                         Spec = new V1PodSpec
                         {
@@ -362,9 +361,9 @@ namespace KubernetesWorkflow
             };
         }
 
-        private IDictionary<string, string> GetSelector()
+        private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
         {
-            return podLabels.GetLabels();
+            return containerRecipes.First().PodLabels.GetLabels();
         }
 
         private IDictionary<string, string> GetRunnerNamespaceSelector()
@@ -372,13 +371,19 @@ namespace KubernetesWorkflow
             return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
         }
 
-        private V1ObjectMeta CreateDeploymentMetadata()
+        private IDictionary<string, string> GetAnnotations(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.First().PodAnnotations.GetAnnotations();
+        }
+
+        private V1ObjectMeta CreateDeploymentMetadata(ContainerRecipe[] containerRecipes)
         {
             return new V1ObjectMeta
             {
                 Name = "deploy-" + workflowNumberSource.WorkflowNumber,
                 NamespaceProperty = K8sTestNamespace,
-                Labels = GetSelector()
+                Labels = GetSelector(containerRecipes),
+                Annotations = GetAnnotations(containerRecipes)
             };
         }
 
@@ -458,7 +463,7 @@ namespace KubernetesWorkflow
                 Spec = new V1ServiceSpec
                 {
                     Type = "NodePort",
-                    Selector = GetSelector(),
+                    Selector = GetSelector(containerRecipes),
                     Ports = ports
                 }
             };

KubernetesWorkflow/PodAnnotations.cs

@@ -0,0 +1,29 @@
+namespace KubernetesWorkflow
+{
+    public class PodAnnotations
+    {
+        private readonly Dictionary<string, string> annotations = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            annotations.Add(key, value);
+        }
+
+        public PodAnnotations Clone()
+        {
+            var result = new PodAnnotations();
+            foreach (var entry in annotations) result.Add(entry.Key, entry.Value);
+            return result;
+        }
+
+        public void Clear()
+        {
+            annotations.Clear();
+        }
+
+        internal Dictionary<string, string> GetAnnotations()
+        {
+            return annotations;
+        }
+    }
+}

KubernetesWorkflow/PodLabels.cs

@@ -1,45 +1,26 @@
-using Logging;
+namespace KubernetesWorkflow
-
-namespace KubernetesWorkflow
 {
     public class PodLabels
     {
         private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
 
-        private PodLabels(PodLabels source)
+        public void Add(string key, string value)
-        {
-            labels = source.labels.ToDictionary(p => p.Key, p => p.Value);
-        }
-
-        public PodLabels(string testsType, ApplicationIds applicationIds)
-        {
-            Add("tests-type", testsType);
-            Add("runid", NameUtils.GetRunId());
-            Add("testid", NameUtils.GetTestId());
-            Add("category", NameUtils.GetCategoryName());
-            Add("fixturename", NameUtils.GetRawFixtureName());
-            Add("testname", NameUtils.GetTestMethodName());
-
-            if (applicationIds == null) return;
-            Add("codexid", applicationIds.CodexId);
-            Add("gethid", applicationIds.GethId);
-            Add("prometheusid", applicationIds.PrometheusId);
-            Add("codexcontractsid", applicationIds.CodexContractsId);
-            Add("grafanaid", applicationIds.GrafanaId);
-        }
-
-        public PodLabels GetLabelsForAppName(string appName)
-        {
-            var pl = new PodLabels(this);
-            pl.Add("app", appName);
-            return pl;
-        }
-
-        private void Add(string key, string value)
         {
             labels.Add(key, Format(value));
         }
 
+        public PodLabels Clone()
+        {
+            var result = new PodLabels();
+            foreach (var entry in labels) result.Add(entry.Key, entry.Value);
+            return result;
+        }
+
+        public void Clear()
+        {
+            labels.Clear();
+        }
+
         private static string Format(string s)
         {
             var result = s.ToLowerInvariant()

KubernetesWorkflow/StartupWorkflow.cs

@@ -10,23 +10,19 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownK8SPods;
         private readonly string testNamespace;
-        private readonly PodLabels podLabels;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
 
-        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, PodLabels podLabels)
+        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.knownK8SPods = knownK8SPods;
             this.testNamespace = testNamespace;
-            this.podLabels = podLabels;
         }
 
         public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
-            var pl = podLabels.GetLabelsForAppName(recipeFactory.AppName);
-
             return K8s(controller =>
             {
                 var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
@@ -34,7 +30,7 @@ namespace KubernetesWorkflow
                 var runningPod = controller.BringOnline(recipes, location);
 
                 return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig));
-            }, pl);
+            });
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
@@ -156,22 +152,14 @@ namespace KubernetesWorkflow
 
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
             action(controller);
             controller.Dispose();
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
-            var result = action(controller);
-            controller.Dispose();
-            return result;
-        }
-
-        private T K8s<T>(Func<K8sController, T> action, PodLabels labels)
-        {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, labels);
             var result = action(controller);
             controller.Dispose();
             return result;

KubernetesWorkflow/WorkflowCreator.cs

@@ -10,14 +10,12 @@ namespace KubernetesWorkflow
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
         private readonly BaseLog log;
-        private readonly PodLabels podLabels;
         private readonly string testNamespace;
 
-        public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels, string testNamespace)
+        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespace)
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            this.podLabels = podLabels;
             this.testNamespace = testNamespace.ToLowerInvariant();
         }
 
@@ -26,7 +24,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, podLabels);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
         }
     }
 }