status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/labeling-pods'

This commit is contained in:
benbierens 2023-08-10 14:25:43 +02:00
parent 061694f83a 3de8e29198
commit ea767e23ae
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
31 changed files with 204 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CodexNetDeployer/CodexNodeStarter.cs
View File

@ -8,16 +8,14 @@ namespace CodexNetDeployer
public class CodexNodeStarter
{
private readonly Configuration config;
private readonly WorkflowCreator workflowCreator;
private readonly TestLifecycle lifecycle;
private readonly GethStartResult gethResult;
private string bootstrapSpr = "";
private int validatorsLeft;
public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
{
this.config = config;
this.workflowCreator = workflowCreator;
this.lifecycle = lifecycle;
this.gethResult = gethResult;
validatorsLeft = numberOfValidators;
@ -26,7 +24,7 @@ namespace CodexNetDeployer
public RunningContainer? Start(int i)
{
Console.Write($" - {i} = ");
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var workflowStartup = new StartupConfig();
workflowStartup.Add(gethResult);
workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));

4
CodexNetDeployer/Configuration.cs
View File

@ -46,6 +46,10 @@ namespace CodexNetDeployer
[Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
public bool RecordMetrics { get; set; } = false;
[Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
"set this option to override the label value.")]
public string TestsTypePodLabel { get; set; } = "continuous";
public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;

22
CodexNetDeployer/Deployer.cs
View File

@ -21,7 +21,7 @@ namespace CodexNetDeployer
public CodexDeployment Deploy()
{
Log("Initializing...");
var (workflowCreator, lifecycle) = CreateFacilities();
var lifecycle = CreateTestLifecycle();
Log("Preparing configuration...");
// We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
@ -30,7 +30,7 @@ namespace CodexNetDeployer
setup.MetricsEnabled = config.RecordMetrics;
Log("Creating Geth instance and deploying contracts...");
var gethStarter = new GethStarter(lifecycle, workflowCreator);
var gethStarter = new GethStarter(lifecycle);
var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
Log("Geth started. Codex contracts deployed.");
@ -44,7 +44,7 @@ namespace CodexNetDeployer
Log("Starting Codex nodes...");
// Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value);
var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
var codexContainers = new List<RunningContainer>();
for (var i = 0; i < config.NumberOfCodexNodes; i++)
{
@ -57,7 +57,7 @@ namespace CodexNetDeployer
return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
}
private (WorkflowCreator, TestLifecycle) CreateFacilities()
private TestLifecycle CreateTestLifecycle()
{
var kubeConfig = GetKubeConfig(config.KubeConfigFile);
@ -68,19 +68,11 @@ namespace CodexNetDeployer
logDebug: false,
dataFilesPath: "notUsed",
codexLogLevel: config.CodexLogLevel,
runnerLocation: config.RunnerLocation
runnerLocation: config.RunnerLocation,
k8sNamespacePrefix: config.KubeNamespace
);
var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: config.KubeNamespace,
kubeConfigFile: kubeConfig,
operationTimeout: timeset.K8sOperationTimeout(),
retryDelay: timeset.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
return (workflowCreator, lifecycle);
return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
}
private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)

4
CodexNetDeployer/deploy-continuous-testnet.sh
View File

@ -9,5 +9,5 @@ dotnet run \
--min-price=1024 \
--max-collateral=1024 \
--max-duration=3600000 \
--block-ttl=120 \
-y
--block-ttl=300 \
--record-metrics=true

2
CodexNetDownloader/Program.cs
View File

@ -25,7 +25,7 @@ public class Program
if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
var k8sFactory = new K8sFactory();
var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
foreach (var container in config.CodexDeployment.CodexContainers)
{

6
ContinuousTests/ContinuousTestRunner.cs
View File

@ -60,8 +60,8 @@ namespace ContinuousTests
if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
workflowCreator.CreateWorkflow().DeleteTestResources();
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
}
private void StartLogDownloader(TaskFactory taskFactory)
@ -71,7 +71,7 @@ namespace ContinuousTests
var path = Path.Combine(config.LogPath, "containers");
if (!Directory.Exists(path)) Directory.CreateDirectory(path);
var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
taskFactory.Run(downloader.Run);

17
ContinuousTests/K8sFactory.cs
View File

@ -1,13 +1,12 @@
using DistTestCore.Codex;
using DistTestCore;
using KubernetesWorkflow;
using Logging;
namespace ContinuousTests
{
public class K8sFactory
{
public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
{
var kubeConfig = GetKubeConfig(kubeConfigFile);
var lifecycleConfig = new DistTestCore.Configuration
@ -17,19 +16,11 @@ namespace ContinuousTests
logDebug: false,
dataFilesPath: dataFilePath,
codexLogLevel: CodexLogLevel.Debug,
runnerLocation: runnerLocation
runnerLocation: runnerLocation,
k8sNamespacePrefix: customNamespace
);
var kubeFlowConfig = new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: customNamespace,
kubeConfigFile: kubeConfig,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay());
var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
return (workflowCreator, lifecycle);
return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty);
}
private static string? GetKubeConfig(string kubeConfigFile)

8
ContinuousTests/NodeRunner.cs
View File

@ -40,8 +40,8 @@ namespace ContinuousTests
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
{
var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow();
var lifecycle = CreateTestLifecycle();
var flow = lifecycle.WorkflowCreator.CreateWorkflow();
try
{
@ -89,9 +89,9 @@ namespace ContinuousTests
}
}
private (WorkflowCreator, TestLifecycle) CreateFacilities()
private TestLifecycle CreateTestLifecycle()
{
return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
}
}
}

4
ContinuousTests/SingleTestRun.cs
View File

@ -138,7 +138,7 @@ namespace ContinuousTests
private void DownloadClusterLogs()
{
var k8sFactory = new K8sFactory();
var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
foreach (var container in config.CodexDeployment.CodexContainers)
{
@ -221,7 +221,7 @@ namespace ContinuousTests
private DistTestCore.Configuration CreateFileManagerConfiguration()
{
return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
CodexLogLevel.Error, config.RunnerLocation);
CodexLogLevel.Error, config.RunnerLocation, string.Empty);
}
}
}

7
DistTestCore/BaseStarter.cs
View File

@ -1,18 +1,15 @@
using KubernetesWorkflow;
using Logging;
using Logging;
namespace DistTestCore
{
public class BaseStarter
{
protected readonly TestLifecycle lifecycle;
protected readonly WorkflowCreator workflowCreator;
private Stopwatch? stopwatch;
public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
public BaseStarter(TestLifecycle lifecycle)
{
this.lifecycle = lifecycle;
this.workflowCreator = workflowCreator;
}
protected void LogStart(string msg)

1
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -14,6 +14,7 @@ namespace DistTestCore.Codex
public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
public override string AppName => "codex";
public override string Image { get; }
public CodexContainerRecipe()

6
DistTestCore/CodexStarter.cs
View File

@ -8,8 +8,8 @@ namespace DistTestCore
{
public class CodexStarter : BaseStarter
{
public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public CodexStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -121,7 +121,7 @@ namespace DistTestCore
private StartupWorkflow CreateWorkflow()
{
return workflowCreator.CreateWorkflow();
return lifecycle.WorkflowCreator.CreateWorkflow();
}
private void LogSeparator()

7
DistTestCore/Configuration.cs
View File

@ -12,6 +12,7 @@ namespace DistTestCore
private readonly string dataFilesPath;
private readonly CodexLogLevel codexLogLevel;
private readonly TestRunnerLocation runnerLocation;
private readonly string k8sNamespacePrefix;
public Configuration()
{
@ -21,9 +22,10 @@ namespace DistTestCore
dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
k8sNamespacePrefix = "ct-";
}
public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation, string k8sNamespacePrefix)
{
this.kubeConfigFile = kubeConfigFile;
this.logPath = logPath;
@ -31,12 +33,13 @@ namespace DistTestCore
this.dataFilesPath = dataFilesPath;
this.codexLogLevel = codexLogLevel;
this.runnerLocation = runnerLocation;
this.k8sNamespacePrefix = k8sNamespacePrefix;
}
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{
return new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: "ct-",
k8sNamespacePrefix: k8sNamespacePrefix,
kubeConfigFile: kubeConfigFile,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay()

17
DistTestCore/DistTest.cs
View File

@ -13,6 +13,7 @@ namespace DistTestCore
[Parallelizable(ParallelScope.All)]
public abstract class DistTest
{
private const string TestsType = "dist-tests";
private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies;
private readonly FixtureLog fixtureLog;
@ -52,7 +53,7 @@ namespace DistTestCore
{
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty);
wc.CreateWorkflow().DeleteAllResources();
});
}
@ -195,7 +196,9 @@ namespace DistTestCore
{
lock (lifecycleLock)
{
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
var testNamespace = Guid.NewGuid().ToString();
var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace);
lifecycles.Add(testName, lifecycle);
}
});
}
@ -206,7 +209,7 @@ namespace DistTestCore
var testResult = GetTestResult();
var testDuration = lifecycle.GetTestDuration();
fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
statusLog.ConcludeTest(testResult, testDuration, GetCodexId(lifecycle));
statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
@ -216,14 +219,6 @@ namespace DistTestCore
});
}
private static string GetCodexId(TestLifecycle lifecycle)
{
var v = lifecycle.CodexVersion;
if (v == null) return new CodexContainerRecipe().Image;
if (v.version != "untagged build") return v.version;
return v.revision;
}
private ITimeSet GetTimeSet()
{
if (ShouldUseLongTimeouts()) return new LongTimeSet();

11
DistTestCore/GethStarter.cs
View File

@ -1,5 +1,4 @@
using DistTestCore.Marketplace;
using KubernetesWorkflow;
namespace DistTestCore
{
@ -8,13 +7,13 @@ namespace DistTestCore
private readonly MarketplaceNetworkCache marketplaceNetworkCache;
private readonly GethCompanionNodeStarter companionNodeStarter;
public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public GethStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
marketplaceNetworkCache = new MarketplaceNetworkCache(
new GethBootstrapNodeStarter(lifecycle, workflowCreator),
new CodexContractsStarter(lifecycle, workflowCreator));
companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
new GethBootstrapNodeStarter(lifecycle),
new CodexContractsStarter(lifecycle));
companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
}
public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)

8
DistTestCore/Marketplace/CodexContractsContainerRecipe.cs
View File

@ -7,12 +7,8 @@ namespace DistTestCore.Marketplace
public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
public override string Image { get; }
public CodexContractsContainerRecipe()
{
Image = "codexstorage/dist-tests-codex-contracts-eth:sha-d6fbfdc";
}
public override string AppName => "codex-contracts";
public override string Image => "codexstorage/dist-tests-codex-contracts-eth:sha-d6fbfdc";
protected override void Initialize(StartupConfig startupConfig)
{

6
DistTestCore/Marketplace/CodexContractsStarter.cs
View File

@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace
public class CodexContractsStarter : BaseStarter
{
public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public CodexContractsStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace
{
LogStart("Deploying Codex Marketplace...");
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);

6
DistTestCore/Marketplace/GethBootstrapNodeStarter.cs
View File

@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace
{
public class GethBootstrapNodeStarter : BaseStarter
{
public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public GethBootstrapNodeStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace
LogStart("Starting Geth bootstrap node...");
var startupConfig = CreateBootstrapStartupConfig();
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
var bootstrapContainer = containers.Containers[0];

6
DistTestCore/Marketplace/GethCompanionNodeStarter.cs
View File

@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace
{
private int companionAccountIndex = 0;
public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public GethCompanionNodeStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace
var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
var container = containers.Containers[0];

8
DistTestCore/Marketplace/GethContainerRecipe.cs
View File

@ -10,12 +10,8 @@ namespace DistTestCore.Marketplace
public const string DiscoveryPortTag = "disc_port";
public const string AccountsFilename = "accounts.csv";
public override string Image { get; }
public GethContainerRecipe()
{
Image = "codexstorage/dist-tests-geth:sha-b788a2d";
}
public override string AppName => "geth";
public override string Image => "codexstorage/dist-tests-geth:sha-b788a2d";
protected override void Initialize(StartupConfig startupConfig)
{

8
DistTestCore/Metrics/PrometheusContainerRecipe.cs
View File

@ -4,12 +4,8 @@ namespace DistTestCore.Metrics
{
public class PrometheusContainerRecipe : ContainerRecipeFactory
{
public override string Image { get; }
public PrometheusContainerRecipe()
{
Image = "codexstorage/dist-tests-prometheus:sha-f97d7fd";
}
public override string AppName => "prometheus";
public override string Image => "codexstorage/dist-tests-prometheus:sha-f97d7fd";
protected override void Initialize(StartupConfig startupConfig)
{

6
DistTestCore/PrometheusStarter.cs
View File

@ -7,8 +7,8 @@ namespace DistTestCore
{
public class PrometheusStarter : BaseStarter
{
public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
: base(lifecycle, workflowCreator)
public PrometheusStarter(TestLifecycle lifecycle)
: base(lifecycle)
{
}
@ -18,7 +18,7 @@ namespace DistTestCore
var startupConfig = new StartupConfig();
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
var workflow = workflowCreator.CreateWorkflow();
var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");

37
DistTestCore/TestLifecycle.cs
View File

@ -1,5 +1,7 @@
using DistTestCore.Codex;
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
using KubernetesWorkflow;
using Logging;
using Utils;
@ -10,21 +12,19 @@ namespace DistTestCore
{
private readonly DateTime testStart;
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
: this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
{
}
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace)
{
Log = log;
Configuration = configuration;
TimeSet = timeSet;
var podLabels = new PodLabels(testsType, GetApplicationIds());
WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace);
FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this, workflowCreator);
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator);
CodexStarter = new CodexStarter(this);
PrometheusStarter = new PrometheusStarter(this);
GethStarter = new GethStarter(this);
testStart = DateTime.UtcNow;
CodexVersion = null;
@ -34,6 +34,7 @@ namespace DistTestCore
public BaseLog Log { get; }
public Configuration Configuration { get; }
public ITimeSet TimeSet { get; }
public WorkflowCreator WorkflowCreator { get; }
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }
@ -68,5 +69,23 @@ namespace DistTestCore
{
if (CodexVersion == null) CodexVersion = version;
}
public ApplicationIds GetApplicationIds()
{
return new ApplicationIds(
codexId: GetCodexId(),
gethId: new GethContainerRecipe().Image,
prometheusId: new PrometheusContainerRecipe().Image,
codexContractsId: new CodexContractsContainerRecipe().Image
);
}
private string GetCodexId()
{
var v = CodexVersion;
if (v == null) return new CodexContainerRecipe().Image;
if (v.version != "untagged build") return v.version;
return v.revision;
}
}
}

1
KubernetesWorkflow/ContainerRecipeFactory.cs
View File

@ -27,6 +27,7 @@
return recipe;
}
public abstract string AppName { get; }
public abstract string Image { get; }
protected int ContainerNumber { get; private set; } = 0;
protected int Index { get; private set; } = 0;

6
KubernetesWorkflow/K8sController.cs
View File

@ -11,14 +11,16 @@ namespace KubernetesWorkflow
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownPods;
private readonly WorkflowNumberSource workflowNumberSource;
private readonly PodLabels podLabels;
private readonly K8sClient client;
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace, PodLabels podLabels)
{
this.log = log;
this.cluster = cluster;
this.knownPods = knownPods;
this.workflowNumberSource = workflowNumberSource;
this.podLabels = podLabels;
client = new K8sClient(cluster.GetK8sClientConfig());
K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
@ -362,7 +364,7 @@ namespace KubernetesWorkflow
private IDictionary<string, string> GetSelector()
{
return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
return podLabels.GetLabels();
}
private IDictionary<string, string> GetRunnerNamespaceSelector()

52
KubernetesWorkflow/PodLabels.cs Normal file
View File

@ -0,0 +1,52 @@
using Logging;
namespace KubernetesWorkflow
{
public class PodLabels
{
private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
private PodLabels(PodLabels source)
{
labels = source.labels.ToDictionary(p => p.Key, p => p.Value);
}
public PodLabels(string testsType, ApplicationIds applicationIds)
{
Add("tests-type", testsType);
Add("runid", NameUtils.GetRunId());
Add("testid", NameUtils.GetTestId());
Add("category", NameUtils.GetCategoryName());
Add("fixturename", NameUtils.GetRawFixtureName());
Add("testname", NameUtils.GetTestMethodName());
if (applicationIds == null) return;
Add("codexid", applicationIds.CodexId);
Add("gethid", applicationIds.GethId);
Add("prometheusid", applicationIds.PrometheusId);
Add("codexcontractsid", applicationIds.CodexContractsId);
}
public PodLabels GetLabelsForAppName(string appName)
{
var pl = new PodLabels(this);
pl.Add("app", appName);
return pl;
}
private void Add(string key, string value)
{
labels.Add(key,
value.ToLowerInvariant()
.Replace(":","-")
.Replace("/", "-")
.Replace("\\", "-")
);
}
internal Dictionary<string, string> GetLabels()
{
return labels;
}
}
}

20
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -10,19 +10,23 @@ namespace KubernetesWorkflow
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownK8SPods;
private readonly string testNamespace;
private readonly PodLabels podLabels;
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, PodLabels podLabels)
{
this.log = log;
this.numberSource = numberSource;
this.cluster = cluster;
this.knownK8SPods = knownK8SPods;
this.testNamespace = testNamespace;
this.podLabels = podLabels;
}
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
{
var pl = podLabels.GetLabelsForAppName(recipeFactory.AppName);
return K8s(controller =>
{
var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
@ -30,7 +34,7 @@ namespace KubernetesWorkflow
var runningPod = controller.BringOnline(recipes, location);
return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig));
});
}, pl);
}
public void Stop(RunningContainers runningContainers)
@ -147,14 +151,22 @@ namespace KubernetesWorkflow
private void K8s(Action<K8sController> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
action(controller);
controller.Dispose();
}
private T K8s<T>(Func<K8sController, T> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
var result = action(controller);
controller.Dispose();
return result;
}
private T K8s<T>(Func<K8sController, T> action, PodLabels labels)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, labels);
var result = action(controller);
controller.Dispose();
return result;

13
KubernetesWorkflow/WorkflowCreator.cs
View File

@ -10,18 +10,15 @@ namespace KubernetesWorkflow
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly K8sCluster cluster;
private readonly BaseLog log;
private readonly PodLabels podLabels;
private readonly string testNamespace;
public WorkflowCreator(BaseLog log, Configuration configuration)
: this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant())
{
}
public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix)
public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels, string testNamespace)
{
cluster = new K8sCluster(configuration);
this.log = log;
testNamespace = testNamespacePostfix;
this.podLabels = podLabels;
this.testNamespace = testNamespace.ToLowerInvariant();
}
public StartupWorkflow CreateWorkflow()
@ -29,7 +26,7 @@ namespace KubernetesWorkflow
var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
containerNumberSource);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, podLabels);
}
}
}

18
Logging/ApplicationIds.cs Normal file
View File

@ -0,0 +1,18 @@
namespace Logging
{
public class ApplicationIds
{
public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId)
{
CodexId = codexId;
GethId = gethId;
PrometheusId = prometheusId;
CodexContractsId = codexContractsId;
}
public string CodexId { get; }
public string GethId { get; }
public string PrometheusId { get; }
public string CodexContractsId { get; }
}
}

4
Logging/NameUtils.cs
View File

@ -22,6 +22,7 @@ namespace Logging
public static string GetRawFixtureName()
{
var test = TestContext.CurrentContext.Test;
if (test.ClassName!.Contains("AdhocContext")) return "none";
var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
return className.Replace('.', '-');
}
@ -29,6 +30,7 @@ namespace Logging
public static string GetCategoryName()
{
var test = TestContext.CurrentContext.Test;
if (test.ClassName!.Contains("AdhocContext")) return "none";
return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
}
@ -45,7 +47,7 @@ namespace Logging
private static string GetEnvVar(string name)
{
var v = Environment.GetEnvironmentVariable(name);
if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet";
if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet";
return v;
}

10
Logging/StatusLog.cs
View File

@ -14,7 +14,7 @@ namespace Logging
fixtureName = NameUtils.GetRawFixtureName();
}
public void ConcludeTest(string resultStatus, string testDuration, string codexId)
public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds)
{
Write(new StatusLogJson
{
@ -22,7 +22,10 @@ namespace Logging
runid = NameUtils.GetRunId(),
status = resultStatus,
testid = NameUtils.GetTestId(),
codexid = codexId,
codexid = applicationIds.CodexId,
gethid = applicationIds.GethId,
prometheusid = applicationIds.PrometheusId,
codexcontractsid = applicationIds.CodexContractsId,
category = NameUtils.GetCategoryName(),
fixturename = fixtureName,
testname = NameUtils.GetTestMethodName(),
@ -53,6 +56,9 @@ namespace Logging
public string status { get; set; } = string.Empty;
public string testid { get; set; } = string.Empty;
public string codexid { get; set; } = string.Empty;
public string gethid { get; set; } = string.Empty;
public string prometheusid { get; set; } = string.Empty;
public string codexcontractsid { get; set; } = string.Empty;
public string category { get; set; } = string.Empty;
public string fixturename { get; set; } = string.Empty;
public string testname { get; set; } = string.Empty;