status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

simple test lined up

This commit is contained in:
benbierens 2023-09-12 10:31:55 +02:00
parent 48dda1735c
commit 0f86642524
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
23 changed files with 281 additions and 242 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
CodexPlugin/CodexAccess.cs
View File

@ -1,29 +1,24 @@
using DistTestCore;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace CodexPlugin
{
public class CodexAccess : ILogHandler
{
private readonly ILog log;
private readonly ITimeSet timeSet;
private readonly IPluginTools tools;
private bool hasContainerCrashed;
public CodexAccess(ILog log, RunningContainer container, ITimeSet timeSet, Address address)
public CodexAccess(IPluginTools tools, RunningContainer container)
{
this.log = log;
this.tools = tools;
Container = container;
this.timeSet = timeSet;
Address = address;
hasContainerCrashed = false;
if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
}
public RunningContainer Container { get; }
public Address Address { get; }
public CodexDebugResponse GetDebugInfo()
{
@ -90,7 +85,7 @@ namespace CodexPlugin
private Http Http()
{
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
}
private void CheckContainerCrashed(HttpClient client)
@ -100,6 +95,7 @@ namespace CodexPlugin
public void Log(Stream crashLog)
{
var log = tools.GetLog();
var file = log.CreateSubfile();
log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");

13
CodexPlugin/CodexNodeGroup.cs
View File

@ -13,14 +13,10 @@ namespace CodexPlugin
public class CodexNodeGroup : ICodexNodeGroup
{
//private readonly TestLifecycle lifecycle;
public CodexNodeGroup(/*TestLifecycle lifecycle, CodexSetup setup,*/ILog log, ITimeSet timeSet, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
public CodexNodeGroup(IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
{
//this.lifecycle = lifecycle;
//Setup = setup;
Containers = containers;
Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, log, timeSet, codexNodeFactory)).ToArray();
Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
Version = new CodexDebugVersionResponse();
}
@ -43,7 +39,6 @@ namespace CodexPlugin
Containers = null!;
}
//public CodexSetup Setup { get; private set; }
public RunningContainers[] Containers { get; private set; }
public OnlineCodexNode[] Nodes { get; private set; }
public CodexDebugVersionResponse Version { get; private set; }
@ -78,9 +73,9 @@ namespace CodexPlugin
Version = first;
}
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ILog log, ITimeSet timeSet, ICodexNodeFactory factory)
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory)
{
var access = new CodexAccess(log, c, timeSet, c.Address);
var access = new CodexAccess(tools, c);
return factory.CreateOnlineCodexNode(access, this);
}
}

16
CodexPlugin/CodexStarter.cs
View File

@ -6,16 +6,16 @@ namespace CodexPlugin
{
public class CodexStarter
{
private readonly IPluginActions pluginActions;
private readonly IPluginTools pluginTools;
//public CodexStarter(TestLifecycle lifecycle)
// : base(lifecycle)
//{
//}
public CodexStarter(IPluginActions pluginActions)
public CodexStarter(IPluginTools pluginActions)
{
this.pluginActions = pluginActions;
this.pluginTools = pluginActions;
}
public RunningContainers[] BringOnline(CodexSetup codexSetup)
@ -120,19 +120,19 @@ namespace CodexPlugin
var recipe = new CodexContainerRecipe();
for (var i = 0; i < numberOfNodes; i++)
{
var workflow = pluginActions.CreateWorkflow();
var workflow = pluginTools.CreateWorkflow();
result.Add(workflow.Start(1, location, recipe, startupConfig));
}
return result.ToArray();
}
private CodexNodeGroup CreateCodexGroup(/*CodexSetup codexSetup, */RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
private CodexNodeGroup CreateCodexGroup(RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
{
var group = new CodexNodeGroup(pluginActions.GetLog(), pluginActions.GetTimeSet(), /*lifecycle, codexSetup,*/ runningContainers, codexNodeFactory);
var group = new CodexNodeGroup(pluginTools, runningContainers, codexNodeFactory);
try
{
Stopwatch.Measure(pluginActions.GetLog(), "EnsureOnline", group.EnsureOnline, debug: true);
Stopwatch.Measure(pluginTools.GetLog(), "EnsureOnline", group.EnsureOnline, debug: true);
}
catch
{
@ -145,7 +145,7 @@ namespace CodexPlugin
private void CodexNodesNotOnline(RunningContainers[] runningContainers)
{
pluginActions.GetLog().Log("Codex nodes failed to start");
pluginTools.GetLog().Log("Codex nodes failed to start");
// todo:
//foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
}

2
CodexPlugin/DistTestExtensions.cs
View File

@ -12,7 +12,7 @@ namespace CodexPlugin
return Plugin.StartCodexNodes(number, setup);
}
public static ICodexNodeGroup WrapCodexContainers(this DistTest distTest, RunningContainers containers)
public static ICodexNodeGroup WrapCodexContainers(this DistTest distTest, RunningContainers[] containers)
{
return Plugin.WrapCodexContainers(containers);
}

21
CodexPlugin/Plugin.cs
View File

@ -1,19 +1,33 @@
using DistTestCore;
using KubernetesWorkflow;
using Logging;
namespace CodexPlugin
{
public class Plugin : IProjectPlugin
{
private readonly CodexStarter codexStarter;
private CodexStarter codexStarter = null!;
public Plugin(IPluginActions actions)
#region IProjectPlugin Implementation
public void Announce(ILog log)
{
codexStarter = new CodexStarter(actions);
log.Log("hello from codex plugin. codex container info here.");
}
public void Initialize(IPluginTools tools)
{
codexStarter = new CodexStarter(tools);
DistTestExtensions.Plugin = this;
}
public void Finalize(ILog log)
{
}
#endregion
public RunningContainers[] StartCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
{
var codexSetup = new CodexSetup(numberOfNodes, CodexLogLevel.Trace);
@ -36,5 +50,6 @@ namespace CodexPlugin
var rc = StartCodexNodes(1, s => { });
return WrapCodexContainers(rc);
}
}
}

2
ContinuousTests/ContinuousTestRunner.cs
View File

@ -59,7 +59,7 @@ namespace ContinuousTests
log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
lifecycle.WorkflowCreator.CreateWorkflow().DeleteNamespacesStartingWith();
}
}
}

2
ContinuousTests/NodeRunner.cs
View File

@ -101,7 +101,7 @@ namespace ContinuousTests
}
finally
{
flow.DeleteTestResources();
flow.DeleteNamespacesStartingWith();
}
}

1
DistTestCore/Configuration.cs
View File

@ -36,7 +36,6 @@ namespace DistTestCore
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
{
return new KubernetesWorkflow.Configuration(
k8sNamespacePrefix: k8sNamespacePrefix,
kubeConfigFile: kubeConfigFile,
operationTimeout: timeSet.K8sOperationTimeout(),
retryDelay: timeSet.WaitForK8sServiceDelay()

53
DistTestCore/DistTest.cs
View File

@ -18,6 +18,7 @@ namespace DistTestCore
private readonly StatusLog statusLog;
private readonly object lifecycleLock = new object();
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
private readonly PluginManager PluginManager = new PluginManager();
public DistTest()
{
@ -33,11 +34,9 @@ namespace DistTestCore
[OneTimeSetUp]
public void GlobalSetup()
{
fixtureLog.Log($"Codex Distributed Tests are starting...");
//fixtureLog.Log($"Codex image: '{new CodexContainerRecipe().Image}'");
//fixtureLog.Log($"CodexContracts image: '{new CodexContractsContainerRecipe().Image}'");
//fixtureLog.Log($"Prometheus image: '{new PrometheusContainerRecipe().Image}'");
//fixtureLog.Log($"Geth image: '{new GethContainerRecipe().Image}'");
fixtureLog.Log($"Distributed Tests are starting...");
PluginManager.DiscoverPlugins();
AnnouncePlugins(fixtureLog);
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
@ -46,7 +45,7 @@ namespace DistTestCore
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), string.Empty);
wc.CreateWorkflow().DeleteAllResources();
wc.CreateWorkflow().DeleteNamespace();
});
}
catch (Exception ex)
@ -59,6 +58,12 @@ namespace DistTestCore
fixtureLog.Log("Global setup cleanup successful");
}
[OneTimeTearDown]
public void GlobalTearDown()
{
FinalizePlugins(fixtureLog);
}
[SetUp]
public void SetUpDistTest()
{
@ -148,7 +153,17 @@ namespace DistTestCore
// return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
//}
public BaseLog GetTestLog()
private void AnnouncePlugins(FixtureLog fixtureLog)
{
PluginManager.AnnouncePlugins(fixtureLog);
}
private void FinalizePlugins(FixtureLog fixtureLog)
{
PluginManager.FinalizePlugins(fixtureLog);
}
public ILog GetTestLog()
{
return Get().Log;
}
@ -205,7 +220,7 @@ namespace DistTestCore
var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
lifecycles.Add(testName, lifecycle);
DefaultContainerRecipe.TestsType = TestsType;
DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
//DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
}
});
}
@ -216,16 +231,34 @@ namespace DistTestCore
var testResult = GetTestResult();
var testDuration = lifecycle.GetTestDuration();
fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
statusLog.ConcludeTest(testResult, testDuration);//, lifecycle.GetApplicationIds());
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
WriteEndTestLog(lifecycle.Log);
IncludeLogsAndMetricsOnTestFailure(lifecycle);
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private void WriteEndTestLog(TestLog log)
{
var result = TestContext.CurrentContext.Result;
Log($"*** Finished: {GetCurrentTestName()} = {result.Outcome.Status}");
if (!string.IsNullOrEmpty(result.Message))
{
Log(result.Message);
Log($"{result.StackTrace}");
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
log.MarkAsFailed();
}
}
private ITimeSet GetTimeSet()
{
if (ShouldUseLongTimeouts()) return new LongTimeSet();

103
DistTestCore/PluginManager.cs
View File

@ -1,64 +1,85 @@
using KubernetesWorkflow;
using FileUtils;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore
{
public class PluginManager : IPluginActions
public class PluginManager
{
private readonly BaseLog log;
private readonly Configuration configuration;
private readonly string testNamespace;
private readonly WorkflowCreator workflowCreator;
private readonly ITimeSet timeSet;
private readonly List<IProjectPlugin> projectPlugins = new List<IProjectPlugin>();
public PluginManager(BaseLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
public void DiscoverPlugins()
{
this.log = log;
this.configuration = configuration;
this.timeSet = timeSet;
this.testNamespace = testNamespace;
workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace);
}
public IStartupWorkflow CreateWorkflow()
{
return workflowCreator.CreateWorkflow();
}
public ILog GetLog()
{
return log;
}
public ITimeSet GetTimeSet()
{
return timeSet;
}
public void InitializeAllPlugins()
{
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
var pluginTypes = assemblies.SelectMany(a => a.GetTypes().Where(t => typeof(IProjectPlugin).IsAssignableFrom(t))).ToArray();
projectPlugins.Clear();
var pluginTypes = PluginFinder.GetPluginTypes();
foreach (var pluginType in pluginTypes)
{
IPluginActions actions = this;
var plugin = (IProjectPlugin)Activator.CreateInstance(pluginType, args: actions)!;
var plugin = (IProjectPlugin)Activator.CreateInstance(pluginType)!;
projectPlugins.Add(plugin);
}
}
public void AnnouncePlugins(ILog log)
{
foreach (var plugin in projectPlugins) plugin.Announce(log);
}
public void InitializePlugins(IPluginTools tools)
{
foreach (var plugin in projectPlugins) plugin.Initialize(tools);
}
public void FinalizePlugins(ILog log)
{
foreach (var plugin in projectPlugins) plugin.Finalize(log);
}
}
public static class PluginFinder
{
private static Type[]? pluginTypes = null;
public static Type[] GetPluginTypes()
{
if (pluginTypes != null) return pluginTypes;
// Reflection can be costly. Do this only once.
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
pluginTypes = assemblies.SelectMany(a => a.GetTypes().Where(t => typeof(IProjectPlugin).IsAssignableFrom(t))).ToArray();
return pluginTypes;
}
}
public interface IProjectPlugin
{
void Announce(ILog log);
void Initialize(IPluginTools tools);
void Finalize(ILog log);
}
// probably seggregate this out.
public interface IPluginActions
public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
{
}
public interface IWorkflowTool
{
IStartupWorkflow CreateWorkflow(string? namespaceOverride = null);
}
public interface ILogTool
{
IStartupWorkflow CreateWorkflow();
ILog GetLog();
ITimeSet GetTimeSet();
}
public interface IHttpFactoryTool
{
Http CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
Http CreateHttp(Address address, string baseUrl, string? logAlias = null);
}
public interface IFileTool
{
IFileManager GetFileManager();
}
}

133
DistTestCore/TestLifecycle.cs
View File

@ -1,70 +1,81 @@
using DistTestCore.Logs;
using FileUtils;
using FileUtils;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore
{
public class TestLifecycle
public class TestLifecycle : IPluginTools
{
private readonly PluginManager pluginManager;
private readonly DateTime testStart;
public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
{
Log = log;
Configuration = configuration;
TimeSet = timeSet;
WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace);
FileManager = new FileManager(Log, configuration.GetFileManagerFolder());
//CodexStarter = new CodexStarter(this);
PrometheusStarter = new PrometheusStarter(this);
GrafanaStarter = new GrafanaStarter(this);
//GethStarter = new GethStarter(this);
TestNamespace = testNamespace;
testStart = DateTime.UtcNow;
//CodexVersion = null;
FileManager = new FileManager(Log, Configuration.GetFileManagerFolder());
// the plugin manager is starting to look like the testlifecycle, that's bad because they are not supposed to be doing the same things:
// pluginmanager should be useful for disttest-deployer-continuoustest, everyone!
// but testlifecycle should be a disttest specific user of the plugin manager.
// disttest requires a hook by which it can keep track of containers created?? (does it?) /namespace used? for the purpose of cleaning up.
pluginManager = new PluginManager();
pluginManager.DiscoverPlugins();
pluginManager.InitializePlugins(this);
//var pluginManager = new PluginManager(Log, configuration, timeSet, testNamespace);
//pluginManager.InitializeAllPlugins();
Log.WriteLogTag();
log.WriteLogTag();
}
public BaseLog Log { get; }
public TestLog Log { get; }
public Configuration Configuration { get; }
public ITimeSet TimeSet { get; }
public WorkflowCreator WorkflowCreator { get; }
public FileManager FileManager { get; }
//public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }
public GrafanaStarter GrafanaStarter { get; }
//public GethStarter GethStarter { get; }
//public CodexDebugVersionResponse? CodexVersion { get; private set; }
public string TestNamespace { get; }
public IFileManager FileManager { get; }
public Http CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
{
return new Http(Log, TimeSet, address, baseUrl, onClientCreated, logAlias);
}
public Http CreateHttp(Address address, string baseUrl, string? logAlias = null)
{
return new Http(Log, TimeSet, address, baseUrl, logAlias);
}
public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
{
if (namespaceOverride != null) throw new Exception("Namespace override is not supported in the DistTest environment. (It would mess up automatic resource cleanup.)");
var wc = new WorkflowCreator(Log, Configuration.GetK8sConfiguration(TimeSet), TestNamespace);
return wc.CreateWorkflow();
}
public IFileManager GetFileManager()
{
return FileManager;
}
public ILog GetLog()
{
return Log;
}
public void DeleteAllResources()
{
//CodexStarter.DeleteAllResources();
CreateWorkflow().DeleteNamespace();
FileManager.DeleteAllTestFiles();
}
public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
{
var subFile = Log.CreateSubfile();
var description = container.Name;
var handler = new LogDownloadHandler(container, description, subFile);
//public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
//{
// var subFile = Log.CreateSubfile();
// var description = container.Name;
// var handler = new LogDownloadHandler(container, description, subFile);
Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
//CodexStarter.DownloadLog(container, handler, tailLines);
// Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
// //CodexStarter.DownloadLog(container, handler, tailLines);
return new DownloadedLog(subFile, description);
}
// return new DownloadedLog(subFile, description);
//}
public string GetTestDuration()
{
@ -72,30 +83,30 @@ namespace DistTestCore
return Time.FormatDuration(testDuration);
}
//public void SetCodexVersion(CodexDebugVersionResponse version)
////public void SetCodexVersion(CodexDebugVersionResponse version)
////{
//// if (CodexVersion == null) CodexVersion = version;
////}
//public ApplicationIds GetApplicationIds()
//{
// if (CodexVersion == null) CodexVersion = version;
// //return new ApplicationIds(
// // codexId: GetCodexId(),
// // gethId: new GethContainerRecipe().Image,
// // prometheusId: new PrometheusContainerRecipe().Image,
// // codexContractsId: new CodexContractsContainerRecipe().Image,
// // grafanaId: new GrafanaContainerRecipe().Image
// //);
// return null!;
//}
public ApplicationIds GetApplicationIds()
{
//return new ApplicationIds(
// codexId: GetCodexId(),
// gethId: new GethContainerRecipe().Image,
// prometheusId: new PrometheusContainerRecipe().Image,
// codexContractsId: new CodexContractsContainerRecipe().Image,
// grafanaId: new GrafanaContainerRecipe().Image
//);
return null!;
}
private string GetCodexId()
{
return "";
//var v = CodexVersion;
//if (v == null) return new CodexContainerRecipe().Image;
//if (v.version != "untagged build") return v.version;
//return v.revision;
}
//private string GetCodexId()
//{
// return "";
// //var v = CodexVersion;
// //if (v == null) return new CodexContainerRecipe().Image;
// //if (v.version != "untagged build") return v.version;
// //return v.revision;
//}
}
}

4
FileUtils/FileManager.cs
View File

@ -18,11 +18,11 @@ namespace FileUtils
public const int ChunkSize = 1024 * 1024 * 100;
private static NumberSource folderNumberSource = new NumberSource(0);
private readonly Random random = new Random();
private readonly BaseLog log;
private readonly ILog log;
private readonly string folder;
private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
public FileManager(BaseLog log, string rootFolder)
public FileManager(ILog log, string rootFolder)
{
folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));

4
FileUtils/TestFile.cs
View File

@ -6,9 +6,9 @@ namespace FileUtils
{
public class TestFile
{
private readonly BaseLog log;
private readonly ILog log;
public TestFile(BaseLog log, string filename, string label)
public TestFile(ILog log, string filename, string label)
{
this.log = log;
Filename = filename;

4
KubernetesWorkflow/Configuration.cs
View File

@ -2,15 +2,13 @@
{
public class Configuration
{
public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
public Configuration(string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
{
K8sNamespacePrefix = k8sNamespacePrefix;
KubeConfigFile = kubeConfigFile;
OperationTimeout = operationTimeout;
RetryDelay = retryDelay;
}
public string K8sNamespacePrefix { get; }
public string? KubeConfigFile { get; }
public TimeSpan OperationTimeout { get; }
public TimeSpan RetryDelay { get; }

4
KubernetesWorkflow/CrashWatcher.cs
View File

@ -5,7 +5,7 @@ namespace KubernetesWorkflow
{
public class CrashWatcher
{
private readonly BaseLog log;
private readonly ILog log;
private readonly KubernetesClientConfiguration config;
private readonly string k8sNamespace;
private readonly RunningContainer container;
@ -14,7 +14,7 @@ namespace KubernetesWorkflow
private Task? worker;
private Exception? workerException;
public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
public CrashWatcher(ILog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
{
this.log = log;
this.config = config;

56
KubernetesWorkflow/K8sController.cs
View File

@ -7,13 +7,13 @@ namespace KubernetesWorkflow
{
public class K8sController
{
private readonly BaseLog log;
private readonly ILog log;
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownPods;
private readonly WorkflowNumberSource workflowNumberSource;
private readonly K8sClient client;
public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
public K8sController(ILog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
{
this.log = log;
this.cluster = cluster;
@ -21,7 +21,7 @@ namespace KubernetesWorkflow
this.workflowNumberSource = workflowNumberSource;
client = new K8sClient(cluster.GetK8sClientConfig());
K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
K8sNamespace = k8sNamespace;
}
public void Dispose()
@ -54,7 +54,7 @@ namespace KubernetesWorkflow
public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
{
log.Debug();
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sNamespace, recipe.Name, tailLines: tailLines));
logHandler.Log(stream);
}
@ -63,7 +63,7 @@ namespace KubernetesWorkflow
var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})";
log.Debug(cmdAndArgs);
var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
runner.Run();
var result = runner.GetStdOut();
@ -71,12 +71,12 @@ namespace KubernetesWorkflow
return result;
}
public void DeleteAllResources()
public void DeleteAllNamespacesStartingWith(string prefix)
{
log.Debug();
var all = client.Run(c => c.ListNamespace().Items);
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(prefix));
foreach (var ns in namespaces)
{
@ -88,12 +88,12 @@ namespace KubernetesWorkflow
}
}
public void DeleteTestNamespace()
public void DeleteNamespace()
{
log.Debug();
if (IsTestNamespaceOnline())
{
client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0));
client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
}
WaitUntilNamespaceDeleted();
}
@ -145,7 +145,7 @@ namespace KubernetesWorkflow
#region Namespace management
private string K8sTestNamespace { get; }
private string K8sNamespace { get; }
private void EnsureTestNamespace()
{
@ -156,8 +156,8 @@ namespace KubernetesWorkflow
ApiVersion = "v1",
Metadata = new V1ObjectMeta
{
Name = K8sTestNamespace,
Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
Name = K8sNamespace,
Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
}
};
client.Run(c => c.CreateNamespace(namespaceSpec));
@ -168,7 +168,7 @@ namespace KubernetesWorkflow
private bool IsTestNamespaceOnline()
{
return IsNamespaceOnline(K8sTestNamespace);
return IsNamespaceOnline(K8sNamespace);
}
private bool IsNamespaceOnline(string name)
@ -185,7 +185,7 @@ namespace KubernetesWorkflow
Metadata = new V1ObjectMeta
{
Name = "isolate-policy",
NamespaceProperty = K8sTestNamespace
NamespaceProperty = K8sNamespace
},
Spec = new V1NetworkPolicySpec
{
@ -314,7 +314,7 @@ namespace KubernetesWorkflow
}
};
c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace);
c.CreateNamespacedNetworkPolicy(body, K8sNamespace);
});
}
@ -352,7 +352,7 @@ namespace KubernetesWorkflow
}
};
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace));
client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
return deploymentSpec.Metadata.Name;
@ -360,7 +360,7 @@ namespace KubernetesWorkflow
private void DeleteDeployment(string deploymentName)
{
client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace));
client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sNamespace));
WaitUntilDeploymentOffline(deploymentName);
}
@ -400,7 +400,7 @@ namespace KubernetesWorkflow
return new V1ObjectMeta
{
Name = "deploy-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sTestNamespace,
NamespaceProperty = K8sNamespace,
Labels = GetSelector(containerRecipes),
Annotations = GetAnnotations(containerRecipes)
};
@ -495,7 +495,7 @@ namespace KubernetesWorkflow
}
}
}
}, K8sTestNamespace));
}, K8sNamespace));
return new V1Volume
{
@ -571,7 +571,7 @@ namespace KubernetesWorkflow
}
};
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
client.Run(c => c.CreateNamespacedService(serviceSpec, K8sNamespace));
ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
@ -581,7 +581,7 @@ namespace KubernetesWorkflow
private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
{
// For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
foreach (var r in containerRecipes)
{
if (r.ExposedPorts.Any())
@ -610,7 +610,7 @@ namespace KubernetesWorkflow
private void DeleteService(string serviceName)
{
client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
client.Run(c => c.DeleteNamespacedService(serviceName, K8sNamespace));
}
private V1ObjectMeta CreateServiceMetadata()
@ -618,7 +618,7 @@ namespace KubernetesWorkflow
return new V1ObjectMeta
{
Name = "service-" + workflowNumberSource.WorkflowNumber,
NamespaceProperty = K8sTestNamespace
NamespaceProperty = K8sNamespace
};
}
@ -672,7 +672,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace));
var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
});
}
@ -681,7 +681,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace));
var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
return deployment == null || deployment.Status.AvailableReplicas == 0;
});
@ -691,7 +691,7 @@ namespace KubernetesWorkflow
{
WaitUntil(() =>
{
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
return pod == null;
});
@ -714,12 +714,12 @@ namespace KubernetesWorkflow
public CrashWatcher CreateCrashWatcher(RunningContainer container)
{
return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container);
return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sNamespace, container);
}
private PodInfo FetchNewPod()
{
var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");

27
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -9,26 +9,26 @@ namespace KubernetesWorkflow
void Stop(RunningContainers runningContainers);
void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines);
string ExecuteCommand(RunningContainer container, string command, params string[] args);
void DeleteAllResources();// !!! delete namespace then!?
void DeleteTestResources(); // !!! do not mention tests. what are we deleting?
void DeleteNamespace();
void DeleteNamespacesStartingWith(string namespacePrefix);
}
public class StartupWorkflow : IStartupWorkflow
{
private readonly BaseLog log;
private readonly ILog log;
private readonly WorkflowNumberSource numberSource;
private readonly K8sCluster cluster;
private readonly KnownK8sPods knownK8SPods;
private readonly string testNamespace;
private readonly string k8sNamespace;
private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string k8sNamespace)
{
this.log = log;
this.numberSource = numberSource;
this.cluster = cluster;
this.knownK8SPods = knownK8SPods;
this.testNamespace = testNamespace;
this.k8sNamespace = k8sNamespace;
}
public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@ -69,19 +69,19 @@ namespace KubernetesWorkflow
});
}
public void DeleteAllResources()
public void DeleteNamespace()
{
K8s(controller =>
{
controller.DeleteAllResources();
controller.DeleteNamespace();
});
}
public void DeleteTestResources()
public void DeleteNamespacesStartingWith(string namespacePrefix)
{
K8s(controller =>
{
controller.DeleteTestNamespace();
controller.DeleteAllNamespacesStartingWith(namespacePrefix);
});
}
@ -133,11 +133,10 @@ namespace KubernetesWorkflow
private Address GetContainerInternalAddress(ContainerRecipe recipe)
{
var serviceName = "service-" + numberSource.WorkflowNumber;
var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace;
var port = GetInternalPort(recipe);
return new Address(
$"http://{serviceName}.{namespaceName}.svc.cluster.local",
$"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
port);
}
@ -167,14 +166,14 @@ namespace KubernetesWorkflow
private void K8s(Action<K8sController> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
action(controller);
controller.Dispose();
}
private T K8s<T>(Func<K8sController, T> action)
{
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
var result = action(controller);
controller.Dispose();
return result;

4
KubernetesWorkflow/WorkflowCreator.cs
View File

@ -9,10 +9,10 @@ namespace KubernetesWorkflow
private readonly NumberSource containerNumberSource = new NumberSource(0);
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly K8sCluster cluster;
private readonly BaseLog log;
private readonly ILog log;
private readonly string testNamespace;
public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespace)
public WorkflowCreator(ILog log, Configuration configuration, string testNamespace)
{
cluster = new K8sCluster(configuration);
this.log = log;

15
Logging/BaseLog.cs
View File

@ -1,5 +1,4 @@
using System.Diagnostics;
using Utils;
using Utils;
namespace Logging
{
@ -16,7 +15,6 @@ namespace Logging
private readonly NumberSource subfileNumberSource = new NumberSource(0);
private readonly bool debug;
private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
private bool hasFailed;
private LogFile? logFile;
protected BaseLog(bool debug)
@ -35,10 +33,6 @@ namespace Logging
}
}
public virtual void EndTest()
{
}
public virtual void Log(string message)
{
LogFile.Write(ApplyReplacements(message));
@ -59,13 +53,6 @@ namespace Logging
Log($"[ERROR] {message}");
}
public virtual void MarkAsFailed()
{
if (hasFailed) return;
hasFailed = true;
LogFile.ConcatToFilename("_FAILED");
}
public virtual void AddStringReplace(string from, string to)
{
if (string.IsNullOrWhiteSpace(from)) return;

4
Logging/FixtureLog.cs
View File

@ -1,12 +1,12 @@
namespace Logging
{
public class FixtureLog : BaseLog
public class FixtureLog : TestLog
{
private readonly string fullName;
private readonly LogConfig config;
public FixtureLog(LogConfig config, DateTime start, string name = "")
: base(config.DebugEnabled)
: base(config.LogRoot, config.DebugEnabled)
{
fullName = NameUtils.GetFixtureFullName(config, start, name);
this.config = config;

4
Logging/NullLog.cs
View File

@ -26,10 +26,6 @@
Console.WriteLine("Error: " + message);
}
public override void MarkAsFailed()
{
}
public override void AddStringReplace(string from, string to)
{
}

12
Logging/StatusLog.cs
View File

@ -14,7 +14,7 @@ namespace Logging
fixtureName = NameUtils.GetRawFixtureName();
}
public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds)
public void ConcludeTest(string resultStatus, string testDuration/*, ApplicationIds applicationIds*/)
{
Write(new StatusLogJson
{
@ -22,11 +22,11 @@ namespace Logging
runid = NameUtils.GetRunId(),
status = resultStatus,
testid = NameUtils.GetTestId(),
codexid = applicationIds.CodexId,
gethid = applicationIds.GethId,
prometheusid = applicationIds.PrometheusId,
codexcontractsid = applicationIds.CodexContractsId,
grafanaid = applicationIds.GrafanaId,
//codexid = applicationIds.CodexId,
//gethid = applicationIds.GethId,
//prometheusid = applicationIds.PrometheusId,
//codexcontractsid = applicationIds.CodexContractsId,
//grafanaid = applicationIds.GrafanaId,
category = NameUtils.GetCategoryName(),
fixturename = fixtureName,
testname = NameUtils.GetTestMethodName(),

23
Logging/TestLog.cs
View File

@ -1,11 +1,10 @@
using NUnit.Framework;
namespace Logging
namespace Logging
{
public class TestLog : BaseLog
{
private readonly string methodName;
private readonly string fullName;
private bool hasFailed;
public TestLog(string folder, bool debug, string name = "")
: base(debug)
@ -16,21 +15,11 @@ namespace Logging
Log($"*** Begin: {methodName}");
}
public override void EndTest()
public void MarkAsFailed()
{
var result = TestContext.CurrentContext.Result;
Log($"*** Finished: {methodName} = {result.Outcome.Status}");
if (!string.IsNullOrEmpty(result.Message))
{
Log(result.Message);
Log($"{result.StackTrace}");
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
MarkAsFailed();
}
if (hasFailed) return;
hasFailed = true;
LogFile.ConcatToFilename("_FAILED");
}
protected override string GetFullName()