diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 9478a98..0a3b373 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -8,16 +8,14 @@ namespace CodexNetDeployer public class CodexNodeStarter { private readonly Configuration config; - private readonly WorkflowCreator workflowCreator; private readonly TestLifecycle lifecycle; private readonly GethStartResult gethResult; private string bootstrapSpr = ""; private int validatorsLeft; - public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators) + public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators) { this.config = config; - this.workflowCreator = workflowCreator; this.lifecycle = lifecycle; this.gethResult = gethResult; validatorsLeft = numberOfValidators; @@ -26,7 +24,7 @@ namespace CodexNetDeployer public RunningContainer? Start(int i) { Console.Write($" - {i} = "); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var workflowStartup = new StartupConfig(); workflowStartup.Add(gethResult); workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft)); diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index b030618..955084a 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -46,6 +46,10 @@ namespace CodexNetDeployer [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] public bool RecordMetrics { get; set; } = false; + + [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " + + "set this option to override the label value.")] + public string TestsTypePodLabel { get; set; } = "continuous"; public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 4bfa25d..a5e174e 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -21,7 +21,7 @@ namespace CodexNetDeployer public CodexDeployment Deploy() { Log("Initializing..."); - var (workflowCreator, lifecycle) = CreateFacilities(); + var lifecycle = CreateTestLifecycle(); Log("Preparing configuration..."); // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. @@ -30,7 +30,7 @@ namespace CodexNetDeployer setup.MetricsEnabled = config.RecordMetrics; Log("Creating Geth instance and deploying contracts..."); - var gethStarter = new GethStarter(lifecycle, workflowCreator); + var gethStarter = new GethStarter(lifecycle); var gethResults = gethStarter.BringOnlineMarketplaceFor(setup); Log("Geth started. Codex contracts deployed."); @@ -44,7 +44,7 @@ namespace CodexNetDeployer Log("Starting Codex nodes..."); // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. - var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value); + var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value); var codexContainers = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { @@ -57,7 +57,7 @@ namespace CodexNetDeployer return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata()); } - private (WorkflowCreator, TestLifecycle) CreateFacilities() + private TestLifecycle CreateTestLifecycle() { var kubeConfig = GetKubeConfig(config.KubeConfigFile); @@ -68,19 +68,11 @@ namespace CodexNetDeployer logDebug: false, dataFilesPath: "notUsed", codexLogLevel: config.CodexLogLevel, - runnerLocation: config.RunnerLocation + runnerLocation: config.RunnerLocation, + k8sNamespacePrefix: config.KubeNamespace ); - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: config.KubeNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: timeset.K8sOperationTimeout(), - retryDelay: timeset.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); - var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); - - return (workflowCreator, lifecycle); + return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); } private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh index 5337922..338b8c1 100644 --- a/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -9,5 +9,5 @@ dotnet run \ --min-price=1024 \ --max-collateral=1024 \ --max-duration=3600000 \ - --block-ttl=120 \ - -y + --block-ttl=300 \ + --record-metrics=true diff --git a/CodexNetDownloader/Program.cs b/CodexNetDownloader/Program.cs index f52a1d2..195e96b 100644 --- a/CodexNetDownloader/Program.cs +++ b/CodexNetDownloader/Program.cs @@ -25,7 +25,7 @@ public class Program if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath); var k8sFactory = new K8sFactory(); - var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); foreach (var container in config.CodexDeployment.CodexContainers) { diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 985707a..35ff486 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -60,8 +60,8 @@ namespace ContinuousTests if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); - var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); - workflowCreator.CreateWorkflow().DeleteTestResources(); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation); + lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources(); } private void StartLogDownloader(TaskFactory taskFactory) @@ -71,7 +71,7 @@ namespace ContinuousTests var path = Path.Combine(config.LogPath, "containers"); if (!Directory.Exists(path)) Directory.CreateDirectory(path); - var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken); taskFactory.Run(downloader.Run); diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs index 0704e9f..ff9b7e9 100644 --- a/ContinuousTests/K8sFactory.cs +++ b/ContinuousTests/K8sFactory.cs @@ -1,13 +1,12 @@ using DistTestCore.Codex; using DistTestCore; -using KubernetesWorkflow; using Logging; namespace ContinuousTests { public class K8sFactory { - public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) + public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation) { var kubeConfig = GetKubeConfig(kubeConfigFile); var lifecycleConfig = new DistTestCore.Configuration @@ -17,19 +16,11 @@ namespace ContinuousTests logDebug: false, dataFilesPath: dataFilePath, codexLogLevel: CodexLogLevel.Debug, - runnerLocation: runnerLocation + runnerLocation: runnerLocation, + k8sNamespacePrefix: customNamespace ); - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: customNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); - var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator); - - return (workflowCreator, lifecycle); + return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty); } private static string? GetKubeConfig(string kubeConfigFile) diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 02116ee..4bb335e 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -40,8 +40,8 @@ namespace ContinuousTests public void RunNode(CodexAccess bootstrapNode, Action operation, TestToken mintTestTokens) { - var (workflowCreator, lifecycle) = CreateFacilities(); - var flow = workflowCreator.CreateWorkflow(); + var lifecycle = CreateTestLifecycle(); + var flow = lifecycle.WorkflowCreator.CreateWorkflow(); try { @@ -89,9 +89,9 @@ namespace ContinuousTests } } - private (WorkflowCreator, TestLifecycle) CreateFacilities() + private TestLifecycle CreateTestLifecycle() { - return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); + return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation); } } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index b065c77..cff325e 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -138,7 +138,7 @@ namespace ContinuousTests private void DownloadClusterLogs() { var k8sFactory = new K8sFactory(); - var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); + var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation); foreach (var container in config.CodexDeployment.CodexContainers) { @@ -221,7 +221,7 @@ namespace ContinuousTests private DistTestCore.Configuration CreateFileManagerConfiguration() { return new DistTestCore.Configuration(null, string.Empty, false, dataFolder, - CodexLogLevel.Error, config.RunnerLocation); + CodexLogLevel.Error, config.RunnerLocation, string.Empty); } } } diff --git a/DistTestCore/BaseStarter.cs b/DistTestCore/BaseStarter.cs index 83a6b8d..4d10643 100644 --- a/DistTestCore/BaseStarter.cs +++ b/DistTestCore/BaseStarter.cs @@ -1,18 +1,15 @@ -using KubernetesWorkflow; -using Logging; +using Logging; namespace DistTestCore { public class BaseStarter { protected readonly TestLifecycle lifecycle; - protected readonly WorkflowCreator workflowCreator; private Stopwatch? stopwatch; - public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + public BaseStarter(TestLifecycle lifecycle) { this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; } protected void LogStart(string msg) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index ea6f187..c600c01 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -14,6 +14,7 @@ namespace DistTestCore.Codex public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); + public override string AppName => "codex"; public override string Image { get; } public CodexContainerRecipe() diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 690ee94..a1e38b7 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -8,8 +8,8 @@ namespace DistTestCore { public class CodexStarter : BaseStarter { - public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public CodexStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -121,7 +121,7 @@ namespace DistTestCore private StartupWorkflow CreateWorkflow() { - return workflowCreator.CreateWorkflow(); + return lifecycle.WorkflowCreator.CreateWorkflow(); } private void LogSeparator() diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 03514cd..daccad6 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -12,6 +12,7 @@ namespace DistTestCore private readonly string dataFilesPath; private readonly CodexLogLevel codexLogLevel; private readonly TestRunnerLocation runnerLocation; + private readonly string k8sNamespacePrefix; public Configuration() { @@ -21,9 +22,10 @@ namespace DistTestCore dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); + k8sNamespacePrefix = "ct-"; } - public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) + public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix) { this.kubeConfigFile = kubeConfigFile; this.logPath = logPath; @@ -31,12 +33,13 @@ namespace DistTestCore this.dataFilesPath = dataFilesPath; this.codexLogLevel = codexLogLevel; this.runnerLocation = runnerLocation; + this.k8sNamespacePrefix = k8sNamespacePrefix; } public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet) { return new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: "ct-", + k8sNamespacePrefix: k8sNamespacePrefix, kubeConfigFile: kubeConfigFile, operationTimeout: timeSet.K8sOperationTimeout(), retryDelay: timeSet.WaitForK8sServiceDelay() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 6768618..a38377d 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -13,6 +13,7 @@ namespace DistTestCore [Parallelizable(ParallelScope.All)] public abstract class DistTest { + private const string TestsType = "dist-tests"; private readonly Configuration configuration = new Configuration(); private readonly Assembly[] testAssemblies; private readonly FixtureLog fixtureLog; @@ -52,7 +53,7 @@ namespace DistTestCore { Stopwatch.Measure(fixtureLog, "Global setup", () => { - var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet())); + var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty); wc.CreateWorkflow().DeleteAllResources(); }); } @@ -195,7 +196,9 @@ namespace DistTestCore { lock (lifecycleLock) { - lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet())); + var testNamespace = Guid.NewGuid().ToString(); + var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace); + lifecycles.Add(testName, lifecycle); } }); } @@ -206,7 +209,7 @@ namespace DistTestCore var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})"); - statusLog.ConcludeTest(testResult, testDuration, GetCodexId(lifecycle)); + statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds()); Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => { lifecycle.Log.EndTest(); @@ -216,14 +219,6 @@ namespace DistTestCore }); } - private static string GetCodexId(TestLifecycle lifecycle) - { - var v = lifecycle.CodexVersion; - if (v == null) return new CodexContainerRecipe().Image; - if (v.version != "untagged build") return v.version; - return v.revision; - } - private ITimeSet GetTimeSet() { if (ShouldUseLongTimeouts()) return new LongTimeSet(); diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 3b9a9b1..578cc30 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -1,5 +1,4 @@ using DistTestCore.Marketplace; -using KubernetesWorkflow; namespace DistTestCore { @@ -8,13 +7,13 @@ namespace DistTestCore private readonly MarketplaceNetworkCache marketplaceNetworkCache; private readonly GethCompanionNodeStarter companionNodeStarter; - public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethStarter(TestLifecycle lifecycle) + : base(lifecycle) { marketplaceNetworkCache = new MarketplaceNetworkCache( - new GethBootstrapNodeStarter(lifecycle, workflowCreator), - new CodexContractsStarter(lifecycle, workflowCreator)); - companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); + new GethBootstrapNodeStarter(lifecycle), + new CodexContractsStarter(lifecycle)); + companionNodeStarter = new GethCompanionNodeStarter(lifecycle); } public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup) diff --git a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs index b974948..e382703 100644 --- a/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs +++ b/DistTestCore/Marketplace/CodexContractsContainerRecipe.cs @@ -7,12 +7,8 @@ namespace DistTestCore.Marketplace public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; - public override string Image { get; } - - public CodexContractsContainerRecipe() - { - Image = "codexstorage/dist-tests-codex-contracts-eth:sha-d6fbfdc"; - } + public override string AppName => "codex-contracts"; + public override string Image => "codexstorage/dist-tests-codex-contracts-eth:sha-d6fbfdc"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index c41df53..a1db57f 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace public class CodexContractsStarter : BaseStarter { - public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public CodexContractsStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace { LogStart("Deploying Codex Marketplace..."); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]); var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 56296ff..d1ebb54 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeStarter : BaseStarter { - public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethBootstrapNodeStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace LogStart("Starting Geth bootstrap node..."); var startupConfig = CreateBootstrapStartupConfig(); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); var bootstrapContainer = containers.Containers[0]; diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index 2008e9c..6759e7b 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace { private int companionAccountIndex = 0; - public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public GethCompanionNodeStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config)); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 6dc62e9..4ca7895 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -10,12 +10,8 @@ namespace DistTestCore.Marketplace public const string DiscoveryPortTag = "disc_port"; public const string AccountsFilename = "accounts.csv"; - public override string Image { get; } - - public GethContainerRecipe() - { - Image = "codexstorage/dist-tests-geth:sha-b788a2d"; - } + public override string AppName => "geth"; + public override string Image => "codexstorage/dist-tests-geth:sha-b788a2d"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs index b702222..e51fdc5 100644 --- a/DistTestCore/Metrics/PrometheusContainerRecipe.cs +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -4,12 +4,8 @@ namespace DistTestCore.Metrics { public class PrometheusContainerRecipe : ContainerRecipeFactory { - public override string Image { get; } - - public PrometheusContainerRecipe() - { - Image = "codexstorage/dist-tests-prometheus:sha-f97d7fd"; - } + public override string AppName => "prometheus"; + public override string Image => "codexstorage/dist-tests-prometheus:sha-f97d7fd"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 6ef2e57..1804b77 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -7,8 +7,8 @@ namespace DistTestCore { public class PrometheusStarter : BaseStarter { - public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) - : base(lifecycle, workflowCreator) + public PrometheusStarter(TestLifecycle lifecycle) + : base(lifecycle) { } @@ -18,7 +18,7 @@ namespace DistTestCore var startupConfig = new StartupConfig(); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers()))); - var workflow = workflowCreator.CreateWorkflow(); + var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index f33b238..eede32d 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,5 +1,7 @@ using DistTestCore.Codex; using DistTestCore.Logs; +using DistTestCore.Marketplace; +using DistTestCore.Metrics; using KubernetesWorkflow; using Logging; using Utils; @@ -10,21 +12,19 @@ namespace DistTestCore { private readonly DateTime testStart; - public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) - : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet))) - { - } - - public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) + public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace) { Log = log; Configuration = configuration; TimeSet = timeSet; + var podLabels = new PodLabels(testsType, GetApplicationIds()); + WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace); + FileManager = new FileManager(Log, configuration); - CodexStarter = new CodexStarter(this, workflowCreator); - PrometheusStarter = new PrometheusStarter(this, workflowCreator); - GethStarter = new GethStarter(this, workflowCreator); + CodexStarter = new CodexStarter(this); + PrometheusStarter = new PrometheusStarter(this); + GethStarter = new GethStarter(this); testStart = DateTime.UtcNow; CodexVersion = null; @@ -34,6 +34,7 @@ namespace DistTestCore public BaseLog Log { get; } public Configuration Configuration { get; } public ITimeSet TimeSet { get; } + public WorkflowCreator WorkflowCreator { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } @@ -68,5 +69,23 @@ namespace DistTestCore { if (CodexVersion == null) CodexVersion = version; } + + public ApplicationIds GetApplicationIds() + { + return new ApplicationIds( + codexId: GetCodexId(), + gethId: new GethContainerRecipe().Image, + prometheusId: new PrometheusContainerRecipe().Image, + codexContractsId: new CodexContractsContainerRecipe().Image + ); + } + + private string GetCodexId() + { + var v = CodexVersion; + if (v == null) return new CodexContainerRecipe().Image; + if (v.version != "untagged build") return v.version; + return v.revision; + } } } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 340ed2d..470bd42 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -27,6 +27,7 @@ return recipe; } + public abstract string AppName { get; } public abstract string Image { get; } protected int ContainerNumber { get; private set; } = 0; protected int Index { get; private set; } = 0; diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index d679c8c..5569b07 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -11,14 +11,16 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; + private readonly PodLabels podLabels; private readonly K8sClient client; - public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace) + public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace, PodLabels podLabels) { this.log = log; this.cluster = cluster; this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; + this.podLabels = podLabels; client = new K8sClient(cluster.GetK8sClientConfig()); K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace; @@ -362,7 +364,7 @@ namespace KubernetesWorkflow private IDictionary GetSelector() { - return new Dictionary { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } }; + return podLabels.GetLabels(); } private IDictionary GetRunnerNamespaceSelector() diff --git a/KubernetesWorkflow/PodLabels.cs b/KubernetesWorkflow/PodLabels.cs new file mode 100644 index 0000000..60fc332 --- /dev/null +++ b/KubernetesWorkflow/PodLabels.cs @@ -0,0 +1,52 @@ +using Logging; + +namespace KubernetesWorkflow +{ + public class PodLabels + { + private readonly Dictionary labels = new Dictionary(); + + private PodLabels(PodLabels source) + { + labels = source.labels.ToDictionary(p => p.Key, p => p.Value); + } + + public PodLabels(string testsType, ApplicationIds applicationIds) + { + Add("tests-type", testsType); + Add("runid", NameUtils.GetRunId()); + Add("testid", NameUtils.GetTestId()); + Add("category", NameUtils.GetCategoryName()); + Add("fixturename", NameUtils.GetRawFixtureName()); + Add("testname", NameUtils.GetTestMethodName()); + + if (applicationIds == null) return; + Add("codexid", applicationIds.CodexId); + Add("gethid", applicationIds.GethId); + Add("prometheusid", applicationIds.PrometheusId); + Add("codexcontractsid", applicationIds.CodexContractsId); + } + + public PodLabels GetLabelsForAppName(string appName) + { + var pl = new PodLabels(this); + pl.Add("app", appName); + return pl; + } + + private void Add(string key, string value) + { + labels.Add(key, + value.ToLowerInvariant() + .Replace(":","-") + .Replace("/", "-") + .Replace("\\", "-") + ); + } + + internal Dictionary GetLabels() + { + return labels; + } + } +} diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index c7e3bfb..c51b991 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -10,19 +10,23 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownK8SPods; private readonly string testNamespace; + private readonly PodLabels podLabels; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace) + internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, PodLabels podLabels) { this.log = log; this.numberSource = numberSource; this.cluster = cluster; this.knownK8SPods = knownK8SPods; this.testNamespace = testNamespace; + this.podLabels = podLabels; } public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { + var pl = podLabels.GetLabelsForAppName(recipeFactory.AppName); + return K8s(controller => { var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); @@ -30,7 +34,7 @@ namespace KubernetesWorkflow var runningPod = controller.BringOnline(recipes, location); return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig)); - }); + }, pl); } public void Stop(RunningContainers runningContainers) @@ -147,14 +151,22 @@ namespace KubernetesWorkflow private void K8s(Action action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels); action(controller); controller.Dispose(); } private T K8s(Func action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels); + var result = action(controller); + controller.Dispose(); + return result; + } + + private T K8s(Func action, PodLabels labels) + { + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, labels); var result = action(controller); controller.Dispose(); return result; diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index ea1775a..4e1c346 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -10,18 +10,15 @@ namespace KubernetesWorkflow private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; private readonly BaseLog log; + private readonly PodLabels podLabels; private readonly string testNamespace; - public WorkflowCreator(BaseLog log, Configuration configuration) - : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant()) - { - } - - public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix) + public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels, string testNamespace) { cluster = new K8sCluster(configuration); this.log = log; - testNamespace = testNamespacePostfix; + this.podLabels = podLabels; + this.testNamespace = testNamespace.ToLowerInvariant(); } public StartupWorkflow CreateWorkflow() @@ -29,7 +26,7 @@ namespace KubernetesWorkflow var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, podLabels); } } } diff --git a/Logging/ApplicationIds.cs b/Logging/ApplicationIds.cs new file mode 100644 index 0000000..d52dc10 --- /dev/null +++ b/Logging/ApplicationIds.cs @@ -0,0 +1,18 @@ +namespace Logging +{ + public class ApplicationIds + { + public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId) + { + CodexId = codexId; + GethId = gethId; + PrometheusId = prometheusId; + CodexContractsId = codexContractsId; + } + + public string CodexId { get; } + public string GethId { get; } + public string PrometheusId { get; } + public string CodexContractsId { get; } + } +} diff --git a/Logging/NameUtils.cs b/Logging/NameUtils.cs index aee7d84..2ca47e2 100644 --- a/Logging/NameUtils.cs +++ b/Logging/NameUtils.cs @@ -22,6 +22,7 @@ namespace Logging public static string GetRawFixtureName() { var test = TestContext.CurrentContext.Test; + if (test.ClassName!.Contains("AdhocContext")) return "none"; var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); return className.Replace('.', '-'); } @@ -29,6 +30,7 @@ namespace Logging public static string GetCategoryName() { var test = TestContext.CurrentContext.Test; + if (test.ClassName!.Contains("AdhocContext")) return "none"; return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.')); } @@ -45,7 +47,7 @@ namespace Logging private static string GetEnvVar(string name) { var v = Environment.GetEnvironmentVariable(name); - if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet"; + if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet"; return v; } diff --git a/Logging/StatusLog.cs b/Logging/StatusLog.cs index f0a366b..96a8cc2 100644 --- a/Logging/StatusLog.cs +++ b/Logging/StatusLog.cs @@ -14,7 +14,7 @@ namespace Logging fixtureName = NameUtils.GetRawFixtureName(); } - public void ConcludeTest(string resultStatus, string testDuration, string codexId) + public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds) { Write(new StatusLogJson { @@ -22,7 +22,10 @@ namespace Logging runid = NameUtils.GetRunId(), status = resultStatus, testid = NameUtils.GetTestId(), - codexid = codexId, + codexid = applicationIds.CodexId, + gethid = applicationIds.GethId, + prometheusid = applicationIds.PrometheusId, + codexcontractsid = applicationIds.CodexContractsId, category = NameUtils.GetCategoryName(), fixturename = fixtureName, testname = NameUtils.GetTestMethodName(), @@ -53,6 +56,9 @@ namespace Logging public string status { get; set; } = string.Empty; public string testid { get; set; } = string.Empty; public string codexid { get; set; } = string.Empty; + public string gethid { get; set; } = string.Empty; + public string prometheusid { get; set; } = string.Empty; + public string codexcontractsid { get; set; } = string.Empty; public string category { get; set; } = string.Empty; public string fixturename { get; set; } = string.Empty; public string testname { get; set; } = string.Empty;