From 7b91c83f5bc470a6094d32c8c2d321b07e2a2421 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 13:53:55 +0200 Subject: [PATCH] Moving everything around --- CodexDistTestCore/CodexNodeGroup.cs | 10 -- CodexDistTestCore/OfflineCodexNodes.cs | 9 -- CodexDistTestCore/TestLog.cs | 144 ------------------ CodexDistTestCore/Utils.cs | 9 -- DistTestCore/ByteSize.cs | 57 +++++++ DistTestCore/Codex/CodexAccess.cs | 43 ++++++ DistTestCore/Codex/CodexContainerRecipe.cs | 35 +++++ DistTestCore/Codex/CodexLogLevel.cs | 11 ++ DistTestCore/Codex/CodexStartupConfig.cs | 12 ++ DistTestCore/DistTestCore.csproj | 20 +++ DistTestCore/Http.cs | 100 ++++++++++++ DistTestCore/TestLifecycle.cs | 31 ++++ KubernetesWorkflow/ContainerRecipe.cs | 42 +++++ KubernetesWorkflow/ContainerRecipeFactory.cs | 60 ++++++++ KubernetesWorkflow/K8sCluster.cs | 40 +++++ KubernetesWorkflow/K8sController.cs | 25 +++ KubernetesWorkflow/KubernetesWorkflow.csproj | 14 ++ KubernetesWorkflow/Location.cs | 9 ++ KubernetesWorkflow/RecipeComponentFactory.cs | 24 +++ KubernetesWorkflow/RunningContainers.cs | 30 ++++ KubernetesWorkflow/RunningPod.cs | 24 +++ KubernetesWorkflow/StartupConfig.cs | 18 +++ KubernetesWorkflow/StartupWorkflow.cs | 40 +++++ KubernetesWorkflow/WorkflowCreator.cs | 13 ++ .../Config => Logging}/LogConfig.cs | 2 +- Logging/LogFile.cs | 73 +++++++++ Logging/Logging.csproj | 19 +++ Logging/TestLog.cs | 74 +++++++++ LongTests/BasicTests/LargeFileTests.cs | 2 +- LongTests/BasicTests/TestInfraTests.cs | 2 +- .../{LongTests.csproj => TestsLong.csproj} | 0 Utils/NumberSource.cs | 19 +++ Utils/Time.cs | 16 ++ Utils/Utils.csproj | 10 ++ cs-codex-dist-testing.sln | 22 ++- 35 files changed, 882 insertions(+), 177 deletions(-) delete mode 100644 CodexDistTestCore/TestLog.cs create mode 100644 DistTestCore/ByteSize.cs create mode 100644 DistTestCore/Codex/CodexAccess.cs create mode 100644 DistTestCore/Codex/CodexContainerRecipe.cs create mode 100644 DistTestCore/Codex/CodexLogLevel.cs create mode 100644 DistTestCore/Codex/CodexStartupConfig.cs create mode 100644 DistTestCore/DistTestCore.csproj create mode 100644 DistTestCore/Http.cs create mode 100644 DistTestCore/TestLifecycle.cs create mode 100644 KubernetesWorkflow/ContainerRecipe.cs create mode 100644 KubernetesWorkflow/ContainerRecipeFactory.cs create mode 100644 KubernetesWorkflow/K8sCluster.cs create mode 100644 KubernetesWorkflow/K8sController.cs create mode 100644 KubernetesWorkflow/KubernetesWorkflow.csproj create mode 100644 KubernetesWorkflow/Location.cs create mode 100644 KubernetesWorkflow/RecipeComponentFactory.cs create mode 100644 KubernetesWorkflow/RunningContainers.cs create mode 100644 KubernetesWorkflow/RunningPod.cs create mode 100644 KubernetesWorkflow/StartupConfig.cs create mode 100644 KubernetesWorkflow/StartupWorkflow.cs create mode 100644 KubernetesWorkflow/WorkflowCreator.cs rename {CodexDistTestCore/Config => Logging}/LogConfig.cs (72%) create mode 100644 Logging/LogFile.cs create mode 100644 Logging/Logging.csproj create mode 100644 Logging/TestLog.cs rename LongTests/{LongTests.csproj => TestsLong.csproj} (100%) create mode 100644 Utils/NumberSource.cs create mode 100644 Utils/Time.cs create mode 100644 Utils/Utils.csproj diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index 1ad5781..12031ac 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -99,15 +99,5 @@ namespace CodexDistTestCore } } - public class PodInfo - { - public PodInfo(string name, string ip) - { - Name = name; - Ip = ip; - } - public string Name { get; } - public string Ip { get; } - } } diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs index d82ed53..84003a6 100644 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -13,15 +13,6 @@ namespace CodexDistTestCore ICodexNodeGroup BringOnline(); } - public enum CodexLogLevel - { - Trace, - Debug, - Info, - Warn, - Error - } - public enum Location { Unspecified, diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs deleted file mode 100644 index 3947678..0000000 --- a/CodexDistTestCore/TestLog.cs +++ /dev/null @@ -1,144 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public class TestLog - { - private readonly NumberSource subfileNumberSource = new NumberSource(0); - private readonly LogFile file; - private readonly DateTime now; - - public TestLog() - { - now = DateTime.UtcNow; - - var name = GetTestName(); - file = new LogFile(now, name); - - Log($"Begin: {name}"); - } - - public void Log(string message) - { - file.Write(message); - } - - public void Error(string message) - { - Log($"[ERROR] {message}"); - } - - public void EndTest() - { - var result = TestContext.CurrentContext.Result; - - Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); - if (!string.IsNullOrEmpty(result.Message)) - { - Log(result.Message); - Log($"{result.StackTrace}"); - } - - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - RenameLogFile(); - } - } - - private void RenameLogFile() - { - file.ConcatToFilename("_FAILED"); - } - - public LogFile CreateSubfile(string ext = "log") - { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); - } - - private static string GetTestName() - { - var test = TestContext.CurrentContext.Test; - var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); - var args = FormatArguments(test); - return $"{className}.{test.MethodName}{args}"; - } - - private static string FormatArguments(TestContext.TestAdapter test) - { - if (test.Arguments == null || !test.Arguments.Any()) return ""; - return $"[{string.Join(',', test.Arguments)}]"; - } - } - - public class LogFile - { - private readonly DateTime now; - private string name; - private readonly string ext; - private readonly string filepath; - - public LogFile(DateTime now, string name, string ext = "log") - { - this.now = now; - this.name = name; - this.ext = ext; - - filepath = Path.Join( - LogConfig.LogRoot, - $"{now.Year}-{Pad(now.Month)}", - Pad(now.Day)); - - Directory.CreateDirectory(filepath); - - GenerateFilename(); - } - - public string FullFilename { get; private set; } = string.Empty; - public string FilenameWithoutPath { get; private set; } = string.Empty; - - public void Write(string message) - { - WriteRaw($"{GetTimestamp()} {message}"); - } - - public void WriteRaw(string message) - { - try - { - File.AppendAllLines(FullFilename, new[] { message }); - } - catch (Exception ex) - { - Console.WriteLine("Writing to log has failed: " + ex); - } - } - - public void ConcatToFilename(string toAdd) - { - var oldFullName = FullFilename; - - name += toAdd; - - GenerateFilename(); - - File.Move(oldFullName, FullFilename); - } - - private static string Pad(int n) - { - return n.ToString().PadLeft(2, '0'); - } - - private static string GetTimestamp() - { - return $"[{DateTime.UtcNow.ToString("u")}]"; - } - - private void GenerateFilename() - { - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; - FullFilename = Path.Combine(filepath, FilenameWithoutPath); - } - } -} diff --git a/CodexDistTestCore/Utils.cs b/CodexDistTestCore/Utils.cs index e0149a1..ae3e761 100644 --- a/CodexDistTestCore/Utils.cs +++ b/CodexDistTestCore/Utils.cs @@ -2,15 +2,6 @@ { public static class Utils { - public static void Sleep(TimeSpan span) - { - Thread.Sleep(span); - } - public static T Wait(Task task) - { - task.Wait(); - return task.Result; - } } } diff --git a/DistTestCore/ByteSize.cs b/DistTestCore/ByteSize.cs new file mode 100644 index 0000000..4f13da8 --- /dev/null +++ b/DistTestCore/ByteSize.cs @@ -0,0 +1,57 @@ +namespace DistTestCore +{ + public class ByteSize + { + public ByteSize(long sizeInBytes) + { + SizeInBytes = sizeInBytes; + } + + public long SizeInBytes { get; } + } + + public static class IntExtensions + { + private const long Kilo = 1024; + + public static ByteSize KB(this long i) + { + return new ByteSize(i * Kilo); + } + + public static ByteSize MB(this long i) + { + return (i * Kilo).KB(); + } + + public static ByteSize GB(this long i) + { + return (i * Kilo).MB(); + } + + public static ByteSize TB(this long i) + { + return (i * Kilo).GB(); + } + + public static ByteSize KB(this int i) + { + return Convert.ToInt64(i).KB(); + } + + public static ByteSize MB(this int i) + { + return Convert.ToInt64(i).MB(); + } + + public static ByteSize GB(this int i) + { + return Convert.ToInt64(i).GB(); + } + + public static ByteSize TB(this int i) + { + return Convert.ToInt64(i).TB(); + } + } +} diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs new file mode 100644 index 0000000..a7c826e --- /dev/null +++ b/DistTestCore/Codex/CodexAccess.cs @@ -0,0 +1,43 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexAccess + { + private readonly RunningContainer runningContainer; + + public CodexAccess(RunningContainer runningContainer) + { + this.runningContainer = runningContainer; + } + + public CodexDebugResponse GetDebugInfo() + { + var response = Http().HttpGetJson("debug/info"); + //Log($"Got DebugInfo with id: '{response.id}'."); + return response; + } + + private Http Http() + { + var ip = runningContainer.Pod.Cluster.GetIp(); + var port = runningContainer.ServicePorts[0].Number; + return new Http(ip, port, baseUrl: "/api/codex/v1"); + } + } + + public class CodexDebugResponse + { + public string id { get; set; } = string.Empty; + public string[] addrs { get; set; } = new string[0]; + public string repo { get; set; } = string.Empty; + public string spr { get; set; } = string.Empty; + public CodexDebugVersionResponse codex { get; set; } = new(); + } + + public class CodexDebugVersionResponse + { + public string version { get; set; } = string.Empty; + public string revision { get; set; } = string.Empty; + } +} diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs new file mode 100644 index 0000000..9c7b87d --- /dev/null +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -0,0 +1,35 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/nim-codex:sha-b204837"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + AddExposedPortAndVar("API_PORT"); + AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}"); + AddInternalPortAndVar("DISC_PORT"); + + var listenPort = AddInternalPort(); + AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); + + if (config.LogLevel != null) + { + AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); + } + if (config.StorageQuota != null) + { + AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!); + } + if (config.MetricsEnabled) + { + AddEnvVar("METRICS_ADDR", "0.0.0.0"); + AddInternalPortAndVar("METRICS_PORT"); + } + } + } +} diff --git a/DistTestCore/Codex/CodexLogLevel.cs b/DistTestCore/Codex/CodexLogLevel.cs new file mode 100644 index 0000000..cde0eb7 --- /dev/null +++ b/DistTestCore/Codex/CodexLogLevel.cs @@ -0,0 +1,11 @@ +namespace DistTestCore.Codex +{ + public enum CodexLogLevel + { + Trace, + Debug, + Info, + Warn, + Error + } +} diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs new file mode 100644 index 0000000..fdbffcf --- /dev/null +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -0,0 +1,12 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexStartupConfig + { + public Location Location { get; set; } + public CodexLogLevel? LogLevel { get; set; } + public ByteSize? StorageQuota { get; set; } + public bool MetricsEnabled { get; set; } + } +} diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj new file mode 100644 index 0000000..ae8e964 --- /dev/null +++ b/DistTestCore/DistTestCore.csproj @@ -0,0 +1,20 @@ + + + + net6.0 + DistTestCore + enable + enable + + + + + + + + + + + + + diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs new file mode 100644 index 0000000..365920b --- /dev/null +++ b/DistTestCore/Http.cs @@ -0,0 +1,100 @@ +using Newtonsoft.Json; +using NUnit.Framework; +using System.Net.Http.Headers; + +namespace DistTestCore +{ + public class Http + { + private readonly string ip; + private readonly int port; + private readonly string baseUrl; + + public Http(string ip, int port, string baseUrl) + { + this.ip = ip; + this.port = port; + this.baseUrl = baseUrl; + + if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; + if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; + } + + public string HttpGetString(string route) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + var result = Utils.Wait(client.GetAsync(url)); + return Utils.Wait(result.Content.ReadAsStringAsync()); + }); + } + + public T HttpGetJson(string route) + { + return JsonConvert.DeserializeObject(HttpGetString(route))!; + } + + public string HttpPostStream(string route, Stream stream) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + + var content = new StreamContent(stream); + content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + var response = Utils.Wait(client.PostAsync(url, content)); + + return Utils.Wait(response.Content.ReadAsStringAsync()); + }); + } + + public Stream HttpGetStream(string route) + { + return Retry(() => + { + var client = GetClient(); + var url = GetUrl() + route; + + return Utils.Wait(client.GetStreamAsync(url)); + }); + } + + private string GetUrl() + { + return $"http://{ip}:{port}{baseUrl}"; + } + + private static T Retry(Func operation) + { + var retryCounter = 0; + + while (true) + { + try + { + return operation(); + } + catch (Exception exception) + { + Timing.HttpCallRetryDelay(); + retryCounter++; + if (retryCounter > Timing.HttpCallRetryCount()) + { + Assert.Fail(exception.Message); + throw; + } + } + } + } + + private static HttpClient GetClient() + { + var client = new HttpClient(); + client.Timeout = Timing.HttpCallTimeout(); + return client; + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs new file mode 100644 index 0000000..e3d2186 --- /dev/null +++ b/DistTestCore/TestLifecycle.cs @@ -0,0 +1,31 @@ +using DistTestCore.Codex; +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class TestLifecycle + { + private readonly WorkflowCreator workflowCreator = new WorkflowCreator(); + + public void SetUpCodexNodes() + { + var config = new CodexStartupConfig() + { + StorageQuota = 10.MB(), + Location = Location.Unspecified, + LogLevel = CodexLogLevel.Error, + MetricsEnabled = false, + }; + + var workflow = workflowCreator.CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.Add(config); + var containers = workflow.Start(3, new CodexContainerRecipe(), startupConfig); + + foreach (var c in containers.Containers) + { + var access = new CodexAccess(c); + } + } + } +} diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs new file mode 100644 index 0000000..db8420e --- /dev/null +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -0,0 +1,42 @@ +namespace KubernetesWorkflow +{ + public class ContainerRecipe + { + public ContainerRecipe(string name, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) + { + Name = name; + Image = image; + ExposedPorts = exposedPorts; + InternalPorts = internalPorts; + EnvVars = envVars; + } + + public string Name { get; } + public string Image { get; } + public Port[] ExposedPorts { get; } + public Port[] InternalPorts { get; } + public EnvVar[] EnvVars { get; } + } + + public class Port + { + public Port(int number) + { + Number = number; + } + + public int Number { get; } + } + + public class EnvVar + { + public EnvVar(string name, string value) + { + Name = name; + Value = value; + } + + public string Name { get; } + public string Value { get; } + } +} diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs new file mode 100644 index 0000000..d4dfb0a --- /dev/null +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -0,0 +1,60 @@ +namespace KubernetesWorkflow +{ + public abstract class ContainerRecipeFactory + { + private readonly List exposedPorts = new List(); + private readonly List internalPorts = new List(); + private readonly List envVars = new List(); + private RecipeComponentFactory factory = null!; + + public ContainerRecipe CreateRecipe(int containerNumber, RecipeComponentFactory factory, StartupConfig config) + { + this.factory = factory; + ContainerNumber = containerNumber; + + Initialize(config); + + var name = $"ctnr{containerNumber}"; + + return new ContainerRecipe(name, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + } + + protected abstract string Image { get; } + protected int ContainerNumber { get; private set; } = 0; + protected abstract void Initialize(StartupConfig config); + + protected Port AddExposedPort() + { + var p = factory.CreatePort(); + exposedPorts.Add(p); + return p; + } + + protected Port AddInternalPort() + { + var p = factory.CreatePort(); + internalPorts.Add(p); + return p; + } + + protected void AddExposedPortAndVar(string name) + { + AddEnvVar(name, AddExposedPort()); + } + + protected void AddInternalPortAndVar(string name) + { + AddEnvVar(name, AddInternalPort()); + } + + protected void AddEnvVar(string name, string value) + { + envVars.Add(factory.CreateEnvVar(name, value)); + } + + protected void AddEnvVar(string name, Port value) + { + envVars.Add(factory.CreateEnvVar(name, value.Number)); + } + } +} diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs new file mode 100644 index 0000000..d325a75 --- /dev/null +++ b/KubernetesWorkflow/K8sCluster.cs @@ -0,0 +1,40 @@ +using k8s; + +namespace KubernetesWorkflow +{ + public class K8sCluster + { + public const string K8sNamespace = "codex-test-namespace"; + private const string KubeConfigFile = "C:\\kube\\config"; + private readonly Dictionary K8sNodeLocationMap = new Dictionary + { + { Location.BensLaptop, "worker01" }, + { Location.BensOldGamingMachine, "worker02" }, + }; + + private KubernetesClientConfiguration? config; + + public KubernetesClientConfiguration GetK8sClientConfig() + { + if (config != null) return config; + //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); + config = KubernetesClientConfiguration.BuildDefaultConfig(); + return config; + } + + public string GetIp() + { + var c = GetK8sClientConfig(); + + var host = c.Host.Replace("https://", ""); + + return host.Substring(0, host.IndexOf(':')); + } + + public string GetNodeLabelForLocation(Location location) + { + if (location == Location.Unspecified) return string.Empty; + return K8sNodeLocationMap[location]; + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs new file mode 100644 index 0000000..3f6bef6 --- /dev/null +++ b/KubernetesWorkflow/K8sController.cs @@ -0,0 +1,25 @@ +namespace KubernetesWorkflow +{ + public class K8sController + { + private readonly K8sCluster cluster; + + public K8sController(K8sCluster cluster) + { + this.cluster = cluster; + } + + public RunningPod BringOnline(ContainerRecipe[] containerRecipes) + { + // Ensure namespace + // create deployment + // create service if necessary + // wait until deployment online + // fetch pod info + + // for each container, there is now an array of service ports available. + + return null!; + } + } +} diff --git a/KubernetesWorkflow/KubernetesWorkflow.csproj b/KubernetesWorkflow/KubernetesWorkflow.csproj new file mode 100644 index 0000000..557b45c --- /dev/null +++ b/KubernetesWorkflow/KubernetesWorkflow.csproj @@ -0,0 +1,14 @@ + + + + net6.0 + KubernetesWorkflow + enable + enable + + + + + + + diff --git a/KubernetesWorkflow/Location.cs b/KubernetesWorkflow/Location.cs new file mode 100644 index 0000000..3b01284 --- /dev/null +++ b/KubernetesWorkflow/Location.cs @@ -0,0 +1,9 @@ +namespace KubernetesWorkflow +{ + public enum Location + { + Unspecified, + BensLaptop, + BensOldGamingMachine + } +} diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs new file mode 100644 index 0000000..6a40cc9 --- /dev/null +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -0,0 +1,24 @@ +using System.Globalization; + +namespace KubernetesWorkflow +{ + public class RecipeComponentFactory + { + private NumberSource portNumberSource = new NumberSource(8080); + + public Port CreatePort() + { + return new Port(portNumberSource.GetNextNumber()); + } + + public EnvVar CreateEnvVar(string name, int value) + { + return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture)); + } + + public EnvVar CreateEnvVar(string name, string value) + { + return new EnvVar(name, value); + } + } +} diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs new file mode 100644 index 0000000..49fc65f --- /dev/null +++ b/KubernetesWorkflow/RunningContainers.cs @@ -0,0 +1,30 @@ +namespace KubernetesWorkflow +{ + public class RunningContainers + { + public RunningContainers(StartupConfig startupConfig, RunningPod runningPod, RunningContainer[] containers) + { + StartupConfig = startupConfig; + RunningPod = runningPod; + Containers = containers; + } + + public StartupConfig StartupConfig { get; } + public RunningPod RunningPod { get; } + public RunningContainer[] Containers { get; } + } + + public class RunningContainer + { + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts) + { + Pod = pod; + Recipe = recipe; + ServicePorts = servicePorts; + } + + public RunningPod Pod { get; } + public ContainerRecipe Recipe { get; } + public Port[] ServicePorts { get; } + } +} diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs new file mode 100644 index 0000000..63378e2 --- /dev/null +++ b/KubernetesWorkflow/RunningPod.cs @@ -0,0 +1,24 @@ +namespace KubernetesWorkflow +{ + public class RunningPod + { + private readonly Dictionary servicePortMap; + + public RunningPod(K8sCluster cluster, string name, string ip, Dictionary servicePortMap) + { + Cluster = cluster; + Name = name; + Ip = ip; + this.servicePortMap = servicePortMap; + } + + public K8sCluster Cluster { get; } + public string Name { get; } + public string Ip { get; } + + public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) + { + return servicePortMap[containerRecipe]; + } + } +} diff --git a/KubernetesWorkflow/StartupConfig.cs b/KubernetesWorkflow/StartupConfig.cs new file mode 100644 index 0000000..7c13347 --- /dev/null +++ b/KubernetesWorkflow/StartupConfig.cs @@ -0,0 +1,18 @@ +namespace KubernetesWorkflow +{ + public class StartupConfig + { + private readonly List configs = new List(); + + public void Add(object config) + { + configs.Add(config); + } + + public T Get() + { + var match = configs.Single(c => c.GetType() == typeof(T)); + return (T)match; + } + } +} diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs new file mode 100644 index 0000000..eb714f7 --- /dev/null +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -0,0 +1,40 @@ +namespace KubernetesWorkflow +{ + public class StartupWorkflow + { + private readonly NumberSource containerNumberSource; + private readonly K8sController k8SController; + private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); + + public StartupWorkflow(NumberSource containerNumberSource, K8sController k8SController) + { + this.containerNumberSource = containerNumberSource; + this.k8SController = k8SController; + } + + public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + { + var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); + + var runningPod = k8SController.BringOnline(recipes); + + return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes)); + } + + private static RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes) + { + return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r))).ToArray(); + } + + private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + { + var result = new List(); + for (var i = 0; i < numberOfContainers; i++) + { + result.Add(recipeFactory.CreateRecipe(containerNumberSource.GetNextNumber(), componentFactory, startupConfig)); + } + + return result.ToArray(); + } + } +} diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs new file mode 100644 index 0000000..25f5439 --- /dev/null +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -0,0 +1,13 @@ +namespace KubernetesWorkflow +{ + public class WorkflowCreator + { + private readonly NumberSource containerNumberSource = new NumberSource(0); + private readonly K8sController controller = new K8sController(new K8sCluster()); + + public StartupWorkflow CreateWorkflow() + { + return new StartupWorkflow(containerNumberSource, controller); + } + } +} diff --git a/CodexDistTestCore/Config/LogConfig.cs b/Logging/LogConfig.cs similarity index 72% rename from CodexDistTestCore/Config/LogConfig.cs rename to Logging/LogConfig.cs index a14470e..c15b1fe 100644 --- a/CodexDistTestCore/Config/LogConfig.cs +++ b/Logging/LogConfig.cs @@ -1,4 +1,4 @@ -namespace CodexDistTestCore.Config +namespace Logging { public class LogConfig { diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs new file mode 100644 index 0000000..a43186c --- /dev/null +++ b/Logging/LogFile.cs @@ -0,0 +1,73 @@ +namespace Logging +{ + public class LogFile + { + private readonly DateTime now; + private string name; + private readonly string ext; + private readonly string filepath; + + public LogFile(DateTime now, string name, string ext = "log") + { + this.now = now; + this.name = name; + this.ext = ext; + + filepath = Path.Join( + LogConfig.LogRoot, + $"{now.Year}-{Pad(now.Month)}", + Pad(now.Day)); + + Directory.CreateDirectory(filepath); + + GenerateFilename(); + } + + public string FullFilename { get; private set; } = string.Empty; + public string FilenameWithoutPath { get; private set; } = string.Empty; + + public void Write(string message) + { + WriteRaw($"{GetTimestamp()} {message}"); + } + + public void WriteRaw(string message) + { + try + { + File.AppendAllLines(FullFilename, new[] { message }); + } + catch (Exception ex) + { + Console.WriteLine("Writing to log has failed: " + ex); + } + } + + public void ConcatToFilename(string toAdd) + { + var oldFullName = FullFilename; + + name += toAdd; + + GenerateFilename(); + + File.Move(oldFullName, FullFilename); + } + + private static string Pad(int n) + { + return n.ToString().PadLeft(2, '0'); + } + + private static string GetTimestamp() + { + return $"[{DateTime.UtcNow.ToString("u")}]"; + } + + private void GenerateFilename() + { + FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; + FullFilename = Path.Combine(filepath, FilenameWithoutPath); + } + } +} diff --git a/Logging/Logging.csproj b/Logging/Logging.csproj new file mode 100644 index 0000000..fad5ec4 --- /dev/null +++ b/Logging/Logging.csproj @@ -0,0 +1,19 @@ + + + + net6.0 + Logging + enable + enable + + + + + + + + + + + + diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs new file mode 100644 index 0000000..83b6cb1 --- /dev/null +++ b/Logging/TestLog.cs @@ -0,0 +1,74 @@ +using NUnit.Framework; +using Utils; + +namespace Logging +{ + public class TestLog + { + private readonly NumberSource subfileNumberSource = new NumberSource(0); + private readonly LogFile file; + private readonly DateTime now; + + public TestLog() + { + now = DateTime.UtcNow; + + var name = GetTestName(); + file = new LogFile(now, name); + + Log($"Begin: {name}"); + } + + public void Log(string message) + { + file.Write(message); + } + + public void Error(string message) + { + Log($"[ERROR] {message}"); + } + + public void EndTest() + { + var result = TestContext.CurrentContext.Result; + + Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); + if (!string.IsNullOrEmpty(result.Message)) + { + Log(result.Message); + Log($"{result.StackTrace}"); + } + + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + RenameLogFile(); + } + } + + private void RenameLogFile() + { + file.ConcatToFilename("_FAILED"); + } + + public LogFile CreateSubfile(string ext = "log") + { + return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); + } + + private static string GetTestName() + { + var test = TestContext.CurrentContext.Test; + var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); + var args = FormatArguments(test); + return $"{className}.{test.MethodName}{args}"; + } + + private static string FormatArguments(TestContext.TestAdapter test) + { + if (test.Arguments == null || !test.Arguments.Any()) return ""; + return $"[{string.Join(',', test.Arguments)}]"; + } + } + +} diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index b8ab20e..a7f0c9b 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -1,7 +1,7 @@ using CodexDistTestCore; using NUnit.Framework; -namespace LongTests.BasicTests +namespace TestsLong.BasicTests { [TestFixture] public class LargeFileTests : DistTest diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index 705b4fc..c39069c 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -1,7 +1,7 @@ using CodexDistTestCore; using NUnit.Framework; -namespace LongTests.BasicTests +namespace TestsLong.BasicTests { public class TestInfraTests : DistTest { diff --git a/LongTests/LongTests.csproj b/LongTests/TestsLong.csproj similarity index 100% rename from LongTests/LongTests.csproj rename to LongTests/TestsLong.csproj diff --git a/Utils/NumberSource.cs b/Utils/NumberSource.cs new file mode 100644 index 0000000..2c7266f --- /dev/null +++ b/Utils/NumberSource.cs @@ -0,0 +1,19 @@ +namespace Utils +{ + public class NumberSource + { + private int number; + + public NumberSource(int start) + { + number = start; + } + + public int GetNextNumber() + { + var n = number; + number++; + return n; + } + } +} diff --git a/Utils/Time.cs b/Utils/Time.cs new file mode 100644 index 0000000..98c5979 --- /dev/null +++ b/Utils/Time.cs @@ -0,0 +1,16 @@ +namespace Utils +{ + public static class Time + { + public static void Sleep(TimeSpan span) + { + Thread.Sleep(span); + } + + public static T Wait(Task task) + { + task.Wait(); + return task.Result; + } + } +} diff --git a/Utils/Utils.csproj b/Utils/Utils.csproj new file mode 100644 index 0000000..42c2303 --- /dev/null +++ b/Utils/Utils.csproj @@ -0,0 +1,10 @@ + + + + net6.0 + Utils + enable + enable + + + diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 9a7a7ad..43b21e9 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -3,12 +3,18 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.4.33213.308 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{57F57B85-A537-4D3A-B7AE-B72C66B74AAB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj", "{57F57B85-A537-4D3A-B7AE-B72C66B74AAB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LongTests", "LongTests\LongTests.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\TestsLong.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -27,6 +33,18 @@ Global {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.Build.0 = Debug|Any CPU {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.ActiveCfg = Release|Any CPU {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.Build.0 = Release|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.Build.0 = Release|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Release|Any CPU.Build.0 = Release|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE