diff --git a/CodexPlugin/CodexContainerRecipe.cs b/CodexPlugin/CodexContainerRecipe.cs index 08f5eae8..47680a09 100644 --- a/CodexPlugin/CodexContainerRecipe.cs +++ b/CodexPlugin/CodexContainerRecipe.cs @@ -5,7 +5,7 @@ using Utils; namespace CodexPlugin { - public class CodexContainerRecipe : DefaultContainerRecipe + public class CodexContainerRecipe : ContainerRecipeFactory { private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; @@ -27,7 +27,7 @@ namespace CodexPlugin //Resources.Limits = new ContainerResourceSet(milliCPUs: 4000, memory: 12.GB()); } - protected override void InitializeRecipe(StartupConfig startupConfig) + protected override void Initialize(StartupConfig startupConfig) { var config = startupConfig.Get(); @@ -97,6 +97,8 @@ namespace CodexPlugin // AddEnvVar("CODEX_VALIDATOR", "true"); // } //} + + AddPodLabel("codexid", Image); } private ByteSize GetVolumeCapacity(CodexStartupConfig config) diff --git a/CodexPlugin/CodexPlugin.cs b/CodexPlugin/CodexPlugin.cs index e862f784..599ca242 100644 --- a/CodexPlugin/CodexPlugin.cs +++ b/CodexPlugin/CodexPlugin.cs @@ -3,7 +3,7 @@ using KubernetesWorkflow; namespace CodexPlugin { - public class CodexPlugin : IProjectPlugin, IHasLogPrefix + public class CodexPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata { private readonly CodexStarter codexStarter; private readonly IPluginTools tools; @@ -19,7 +19,12 @@ namespace CodexPlugin public void Announce() { - tools.GetLog().Log("hello from codex plugin. codex container info here."); + tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}'"); + } + + public void AddMetadata(IAddMetadata metadata) + { + metadata.Add("codexid", codexStarter.GetCodexId()); } public void Decommission() diff --git a/CodexPlugin/CodexStarter.cs b/CodexPlugin/CodexStarter.cs index fc12369c..69a82e81 100644 --- a/CodexPlugin/CodexStarter.cs +++ b/CodexPlugin/CodexStarter.cs @@ -7,6 +7,8 @@ namespace CodexPlugin public class CodexStarter { private readonly IPluginTools pluginTools; + private readonly CodexContainerRecipe recipe = new CodexContainerRecipe(); + private CodexDebugVersionResponse? versionResponse; public CodexStarter(IPluginTools pluginTools) { @@ -46,8 +48,7 @@ namespace CodexPlugin var group = CreateCodexGroup(containers, codexNodeFactory); Log($"Codex version: {group.Version}"); - - //lifecycle.SetCodexVersion(group.Version); + versionResponse = group.Version; return group; } @@ -64,6 +65,12 @@ namespace CodexPlugin Log("Stopped."); } + public string GetCodexId() + { + if (versionResponse != null) return versionResponse.version; + return recipe.Image; + } + //public void DeleteAllResources() //{ // //var workflow = CreateWorkflow(); @@ -103,7 +110,6 @@ namespace CodexPlugin private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location) { var result = new List(); - var recipe = new CodexContainerRecipe(); for (var i = 0; i < numberOfNodes; i++) { var workflow = pluginTools.CreateWorkflow(); diff --git a/Core/DefaultContainerRecipe.cs b/Core/DefaultContainerRecipe.cs deleted file mode 100644 index 1a34e714..00000000 --- a/Core/DefaultContainerRecipe.cs +++ /dev/null @@ -1,40 +0,0 @@ -using KubernetesWorkflow; -using Logging; - -namespace Core -{ - public abstract class DefaultContainerRecipe : ContainerRecipeFactory - { - public static string TestsType { get; set; } = "NotSet"; - public static ApplicationIds? ApplicationIds { get; set; } = null; - - protected abstract void InitializeRecipe(StartupConfig config); - - protected override void Initialize(StartupConfig config) - { - Add("tests-type", TestsType); - Add("runid", NameUtils.GetRunId()); - Add("testid", NameUtils.GetTestId()); - Add("category", NameUtils.GetCategoryName()); - Add("fixturename", NameUtils.GetRawFixtureName()); - Add("testname", NameUtils.GetTestMethodName()); - - if (ApplicationIds != null) - { - Add("codexid", ApplicationIds.CodexId); - Add("gethid", ApplicationIds.GethId); - Add("prometheusid", ApplicationIds.PrometheusId); - Add("codexcontractsid", ApplicationIds.CodexContractsId); - Add("grafanaid", ApplicationIds.GrafanaId); - } - Add("app", AppName); - - InitializeRecipe(config); - } - - private void Add(string name, string value) - { - AddPodLabel(name, value); - } - } -} diff --git a/Core/EntryPoint.cs b/Core/EntryPoint.cs index 788c15a6..5ce9af1d 100644 --- a/Core/EntryPoint.cs +++ b/Core/EntryPoint.cs @@ -28,6 +28,11 @@ namespace Core manager.AnnouncePlugins(); } + public Dictionary GetPluginMetadata() + { + return manager.GatherPluginMetadata().Get(); + } + public CoreInterface CreateInterface() { return new CoreInterface(this); diff --git a/Core/PluginManager.cs b/Core/PluginManager.cs index 4edccb81..2b247882 100644 --- a/Core/PluginManager.cs +++ b/Core/PluginManager.cs @@ -21,6 +21,19 @@ foreach (var plugin in projectPlugins) plugin.Announce(); } + public PluginMetadata GatherPluginMetadata() + { + var metadata = new PluginMetadata(); + foreach (var plugin in projectPlugins) + { + if (plugin is IHasMetadata m) + { + m.AddMetadata(metadata); + } + } + return metadata; + } + public void DecommissionPlugins() { foreach (var plugin in projectPlugins) plugin.Decommission(); @@ -57,4 +70,9 @@ { string LogPrefix { get; } } + + public interface IHasMetadata + { + void AddMetadata(IAddMetadata metadata); + } } diff --git a/Core/PluginMetadata.cs b/Core/PluginMetadata.cs new file mode 100644 index 00000000..c84af76c --- /dev/null +++ b/Core/PluginMetadata.cs @@ -0,0 +1,27 @@ +namespace Core +{ + public interface IPluginMetadata + { + Dictionary Get(); + } + + public interface IAddMetadata + { + void Add(string key, string value); + } + + public class PluginMetadata : IPluginMetadata, IAddMetadata + { + private readonly Dictionary metadata = new Dictionary(); + + public void Add(string key, string value) + { + metadata.Add(key, value); + } + + public Dictionary Get() + { + return new Dictionary(metadata); + } + } +} diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 408576cd..5555ed80 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -10,7 +10,6 @@ namespace DistTestCore [Parallelizable(ParallelScope.All)] public abstract class DistTest { - private const string TestsType = "dist-tests"; private const string TestNamespacePrefix = "ct-"; private readonly Configuration configuration = new Configuration(); private readonly Assembly[] testAssemblies; @@ -154,8 +153,6 @@ namespace DistTestCore var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace); lifecycles.Add(testName, lifecycle); - DefaultContainerRecipe.TestsType = TestsType; - //DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds(); } }); } @@ -166,7 +163,7 @@ namespace DistTestCore var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})"); - statusLog.ConcludeTest(testResult, testDuration);//, lifecycle.GetApplicationIds()); + statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetPluginMetadata()); Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => { WriteEndTestLog(lifecycle.Log); diff --git a/DistTestCore/StatusLog.cs b/DistTestCore/StatusLog.cs new file mode 100644 index 00000000..e937469b --- /dev/null +++ b/DistTestCore/StatusLog.cs @@ -0,0 +1,46 @@ +using Logging; +using Newtonsoft.Json; + +namespace DistTestCore +{ + public class StatusLog + { + private readonly object fileLock = new object(); + private readonly string fullName; + private readonly string fixtureName; + + public StatusLog(LogConfig config, DateTime start, string name = "") + { + fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log"; + fixtureName = NameUtils.GetRawFixtureName(); + } + + public void ConcludeTest(string resultStatus, string testDuration, Dictionary data) + { + data.Add("timestamp", DateTime.UtcNow.ToString("o")); + data.Add("runid", NameUtils.GetRunId()); + data.Add("status", resultStatus); + data.Add("testid", NameUtils.GetTestId()); + data.Add("category", NameUtils.GetCategoryName()); + data.Add("fixturename", fixtureName); + data.Add("testname", NameUtils.GetTestMethodName()); + data.Add("testduration", testDuration); + Write(data); + } + + private void Write(Dictionary data) + { + try + { + lock (fileLock) + { + File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(data) }); + } + } + catch (Exception ex) + { + Console.WriteLine("Unable to write to status log: " + ex); + } + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 999beb7a..4723dd26 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -8,6 +8,7 @@ namespace DistTestCore { public class TestLifecycle : IK8sHooks { + private const string TestsType = "dist-tests"; private readonly DateTime testStart; private readonly EntryPoint entryPoint; private readonly List runningContainers = new List(); @@ -47,6 +48,11 @@ namespace DistTestCore return entryPoint.Tools.GetFileManager(); } + public Dictionary GetPluginMetadata() + { + return entryPoint.GetPluginMetadata(); + } + public string GetTestDuration() { var testDuration = DateTime.UtcNow - testStart; @@ -63,6 +69,16 @@ namespace DistTestCore runningContainers.Remove(rc); } + public void OnContainerRecipeCreated(ContainerRecipe recipe) + { + recipe.PodLabels.Add("tests-type", TestsType); + recipe.PodLabels.Add("runid", NameUtils.GetRunId()); + recipe.PodLabels.Add("testid", NameUtils.GetTestId()); + recipe.PodLabels.Add("category", NameUtils.GetCategoryName()); + recipe.PodLabels.Add("fixturename", NameUtils.GetRawFixtureName()); + recipe.PodLabels.Add("testname", NameUtils.GetTestMethodName()); + } + public void DownloadAllLogs() { var workflow = entryPoint.Tools.CreateWorkflow(); @@ -82,26 +98,5 @@ namespace DistTestCore var handler = new LogDownloadHandler(c.Name, file); workflow.DownloadContainerLog(c, handler); } - - //public ApplicationIds GetApplicationIds() - //{ - // //return new ApplicationIds( - // // codexId: GetCodexId(), - // // gethId: new GethContainerRecipe().Image, - // // prometheusId: new PrometheusContainerRecipe().Image, - // // codexContractsId: new CodexContractsContainerRecipe().Image, - // // grafanaId: new GrafanaContainerRecipe().Image - // //); - // return null!; - //} - - //private string GetCodexId() - //{ - // return ""; - // //var v = CodexVersion; - // //if (v == null) return new CodexContainerRecipe().Image; - // //if (v.version != "untagged build") return v.version; - // //return v.revision; - //} } } diff --git a/KubernetesWorkflow/Configuration.cs b/KubernetesWorkflow/Configuration.cs index 5cb7da23..1cf30cfc 100644 --- a/KubernetesWorkflow/Configuration.cs +++ b/KubernetesWorkflow/Configuration.cs @@ -15,6 +15,7 @@ public TimeSpan RetryDelay { get; } public string KubernetesNamespace { get; } public bool AllowNamespaceOverride { get; set; } = true; + public bool AddAppPodLabel { get; set; } = true; public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks(); } } diff --git a/KubernetesWorkflow/K8sHooks.cs b/KubernetesWorkflow/K8sHooks.cs index 3bea4d92..91c25a34 100644 --- a/KubernetesWorkflow/K8sHooks.cs +++ b/KubernetesWorkflow/K8sHooks.cs @@ -4,6 +4,7 @@ { void OnContainersStarted(RunningContainers runningContainers); void OnContainersStopped(RunningContainers runningContainers); + void OnContainerRecipeCreated(ContainerRecipe recipe); } public class DoNothingK8sHooks : IK8sHooks @@ -15,5 +16,9 @@ public void OnContainersStopped(RunningContainers runningContainers) { } + + public void OnContainerRecipeCreated(ContainerRecipe recipe) + { + } } } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 3a95262f..d8221d17 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -161,7 +161,10 @@ namespace KubernetesWorkflow var result = new List(); for (var i = 0; i < numberOfContainers; i++) { - result.Add(recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig)); + var recipe = recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig); + if (cluster.Configuration.AddAppPodLabel) recipe.PodLabels.Add("app", recipeFactory.AppName); + cluster.Configuration.Hooks.OnContainerRecipeCreated(recipe); + result.Add(recipe); } return result.ToArray(); diff --git a/Logging/StatusLog.cs b/Logging/StatusLog.cs deleted file mode 100644 index 3fe3a7df..00000000 --- a/Logging/StatusLog.cs +++ /dev/null @@ -1,69 +0,0 @@ -using Newtonsoft.Json; - -namespace Logging -{ - public class StatusLog - { - private readonly object fileLock = new object(); - private readonly string fullName; - private readonly string fixtureName; - - public StatusLog(LogConfig config, DateTime start, string name = "") - { - fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log"; - fixtureName = NameUtils.GetRawFixtureName(); - } - - public void ConcludeTest(string resultStatus, string testDuration/*, ApplicationIds applicationIds*/) - { - Write(new StatusLogJson - { - @timestamp = DateTime.UtcNow.ToString("o"), - runid = NameUtils.GetRunId(), - status = resultStatus, - testid = NameUtils.GetTestId(), - //codexid = applicationIds.CodexId, - //gethid = applicationIds.GethId, - //prometheusid = applicationIds.PrometheusId, - //codexcontractsid = applicationIds.CodexContractsId, - //grafanaid = applicationIds.GrafanaId, - category = NameUtils.GetCategoryName(), - fixturename = fixtureName, - testname = NameUtils.GetTestMethodName(), - testduration = testDuration - }); - } - - private void Write(StatusLogJson json) - { - try - { - lock (fileLock) - { - File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) }); - } - } - catch (Exception ex) - { - Console.WriteLine("Unable to write to status log: " + ex); - } - } - } - - public class StatusLogJson - { - public string @timestamp { get; set; } = string.Empty; - public string runid { get; set; } = string.Empty; - public string status { get; set; } = string.Empty; - public string testid { get; set; } = string.Empty; - public string codexid { get; set; } = string.Empty; - public string gethid { get; set; } = string.Empty; - public string prometheusid { get; set; } = string.Empty; - public string codexcontractsid { get; set; } = string.Empty; - public string grafanaid { get; set; } = string.Empty; - public string category { get; set; } = string.Empty; - public string fixturename { get; set; } = string.Empty; - public string testname { get; set; } = string.Empty; - public string testduration { get; set;} = string.Empty; - } -} diff --git a/MetricsPlugin/MetricsPlugin.cs b/MetricsPlugin/MetricsPlugin.cs index c2ed11cc..09ad08c2 100644 --- a/MetricsPlugin/MetricsPlugin.cs +++ b/MetricsPlugin/MetricsPlugin.cs @@ -4,7 +4,7 @@ using Logging; namespace MetricsPlugin { - public class MetricsPlugin : IProjectPlugin + public class MetricsPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata { private readonly IPluginTools tools; private readonly PrometheusStarter starter; @@ -15,9 +15,16 @@ namespace MetricsPlugin starter = new PrometheusStarter(tools); } + public string LogPrefix => "(Metrics) "; + public void Announce() { - tools.GetLog().Log("Hi from the metrics plugin."); + tools.GetLog().Log($"Prometheus plugin loaded with '{starter.GetPrometheusId()}'."); + } + + public void AddMetadata(IAddMetadata metadata) + { + metadata.Add("prometheusid", starter.GetPrometheusId()); } public void Decommission() diff --git a/MetricsPlugin/PrometheusContainerRecipe.cs b/MetricsPlugin/PrometheusContainerRecipe.cs index 584ad827..26b9b48a 100644 --- a/MetricsPlugin/PrometheusContainerRecipe.cs +++ b/MetricsPlugin/PrometheusContainerRecipe.cs @@ -1,14 +1,13 @@ -using Core; -using KubernetesWorkflow; +using KubernetesWorkflow; namespace MetricsPlugin { - public class PrometheusContainerRecipe : DefaultContainerRecipe + public class PrometheusContainerRecipe : ContainerRecipeFactory { public override string AppName => "prometheus"; public override string Image => "codexstorage/dist-tests-prometheus:latest"; - protected override void InitializeRecipe(StartupConfig startupConfig) + protected override void Initialize(StartupConfig startupConfig) { var config = startupConfig.Get(); diff --git a/MetricsPlugin/PrometheusStarter.cs b/MetricsPlugin/PrometheusStarter.cs index 37814eab..57b8b3ec 100644 --- a/MetricsPlugin/PrometheusStarter.cs +++ b/MetricsPlugin/PrometheusStarter.cs @@ -6,6 +6,7 @@ namespace MetricsPlugin { public class PrometheusStarter { + private readonly PrometheusContainerRecipe recipe = new PrometheusContainerRecipe(); private readonly IPluginTools tools; public PrometheusStarter(IPluginTools tools) @@ -20,7 +21,7 @@ namespace MetricsPlugin startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets))); var workflow = tools.CreateWorkflow(); - var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); + var runningContainers = workflow.Start(1, Location.Unspecified, recipe, startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); Log("Metrics server started."); @@ -33,6 +34,11 @@ namespace MetricsPlugin return new MetricsAccess(metricsQuery, target); } + public string GetPrometheusId() + { + return recipe.Image; + } + private void Log(string msg) { tools.GetLog().Log(msg);