From 7b91c83f5bc470a6094d32c8c2d321b07e2a2421 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 13:53:55 +0200 Subject: [PATCH 01/21] Moving everything around --- CodexDistTestCore/CodexNodeGroup.cs | 10 -- CodexDistTestCore/OfflineCodexNodes.cs | 9 -- CodexDistTestCore/TestLog.cs | 144 ------------------ CodexDistTestCore/Utils.cs | 9 -- DistTestCore/ByteSize.cs | 57 +++++++ DistTestCore/Codex/CodexAccess.cs | 43 ++++++ DistTestCore/Codex/CodexContainerRecipe.cs | 35 +++++ DistTestCore/Codex/CodexLogLevel.cs | 11 ++ DistTestCore/Codex/CodexStartupConfig.cs | 12 ++ DistTestCore/DistTestCore.csproj | 20 +++ DistTestCore/Http.cs | 100 ++++++++++++ DistTestCore/TestLifecycle.cs | 31 ++++ KubernetesWorkflow/ContainerRecipe.cs | 42 +++++ KubernetesWorkflow/ContainerRecipeFactory.cs | 60 ++++++++ KubernetesWorkflow/K8sCluster.cs | 40 +++++ KubernetesWorkflow/K8sController.cs | 25 +++ KubernetesWorkflow/KubernetesWorkflow.csproj | 14 ++ KubernetesWorkflow/Location.cs | 9 ++ KubernetesWorkflow/RecipeComponentFactory.cs | 24 +++ KubernetesWorkflow/RunningContainers.cs | 30 ++++ KubernetesWorkflow/RunningPod.cs | 24 +++ KubernetesWorkflow/StartupConfig.cs | 18 +++ KubernetesWorkflow/StartupWorkflow.cs | 40 +++++ KubernetesWorkflow/WorkflowCreator.cs | 13 ++ .../Config => Logging}/LogConfig.cs | 2 +- Logging/LogFile.cs | 73 +++++++++ Logging/Logging.csproj | 19 +++ Logging/TestLog.cs | 74 +++++++++ LongTests/BasicTests/LargeFileTests.cs | 2 +- LongTests/BasicTests/TestInfraTests.cs | 2 +- .../{LongTests.csproj => TestsLong.csproj} | 0 Utils/NumberSource.cs | 19 +++ Utils/Time.cs | 16 ++ Utils/Utils.csproj | 10 ++ cs-codex-dist-testing.sln | 22 ++- 35 files changed, 882 insertions(+), 177 deletions(-) delete mode 100644 CodexDistTestCore/TestLog.cs create mode 100644 DistTestCore/ByteSize.cs create mode 100644 DistTestCore/Codex/CodexAccess.cs create mode 100644 DistTestCore/Codex/CodexContainerRecipe.cs create mode 100644 DistTestCore/Codex/CodexLogLevel.cs create mode 100644 DistTestCore/Codex/CodexStartupConfig.cs create mode 100644 DistTestCore/DistTestCore.csproj create mode 100644 DistTestCore/Http.cs create mode 100644 DistTestCore/TestLifecycle.cs create mode 100644 KubernetesWorkflow/ContainerRecipe.cs create mode 100644 KubernetesWorkflow/ContainerRecipeFactory.cs create mode 100644 KubernetesWorkflow/K8sCluster.cs create mode 100644 KubernetesWorkflow/K8sController.cs create mode 100644 KubernetesWorkflow/KubernetesWorkflow.csproj create mode 100644 KubernetesWorkflow/Location.cs create mode 100644 KubernetesWorkflow/RecipeComponentFactory.cs create mode 100644 KubernetesWorkflow/RunningContainers.cs create mode 100644 KubernetesWorkflow/RunningPod.cs create mode 100644 KubernetesWorkflow/StartupConfig.cs create mode 100644 KubernetesWorkflow/StartupWorkflow.cs create mode 100644 KubernetesWorkflow/WorkflowCreator.cs rename {CodexDistTestCore/Config => Logging}/LogConfig.cs (72%) create mode 100644 Logging/LogFile.cs create mode 100644 Logging/Logging.csproj create mode 100644 Logging/TestLog.cs rename LongTests/{LongTests.csproj => TestsLong.csproj} (100%) create mode 100644 Utils/NumberSource.cs create mode 100644 Utils/Time.cs create mode 100644 Utils/Utils.csproj diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index 1ad5781..12031ac 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -99,15 +99,5 @@ namespace CodexDistTestCore } } - public class PodInfo - { - public PodInfo(string name, string ip) - { - Name = name; - Ip = ip; - } - public string Name { get; } - public string Ip { get; } - } } diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs index d82ed53..84003a6 100644 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -13,15 +13,6 @@ namespace CodexDistTestCore ICodexNodeGroup BringOnline(); } - public enum CodexLogLevel - { - Trace, - Debug, - Info, - Warn, - Error - } - public enum Location { Unspecified, diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs deleted file mode 100644 index 3947678..0000000 --- a/CodexDistTestCore/TestLog.cs +++ /dev/null @@ -1,144 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public class TestLog - { - private readonly NumberSource subfileNumberSource = new NumberSource(0); - private readonly LogFile file; - private readonly DateTime now; - - public TestLog() - { - now = DateTime.UtcNow; - - var name = GetTestName(); - file = new LogFile(now, name); - - Log($"Begin: {name}"); - } - - public void Log(string message) - { - file.Write(message); - } - - public void Error(string message) - { - Log($"[ERROR] {message}"); - } - - public void EndTest() - { - var result = TestContext.CurrentContext.Result; - - Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); - if (!string.IsNullOrEmpty(result.Message)) - { - Log(result.Message); - Log($"{result.StackTrace}"); - } - - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - RenameLogFile(); - } - } - - private void RenameLogFile() - { - file.ConcatToFilename("_FAILED"); - } - - public LogFile CreateSubfile(string ext = "log") - { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); - } - - private static string GetTestName() - { - var test = TestContext.CurrentContext.Test; - var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); - var args = FormatArguments(test); - return $"{className}.{test.MethodName}{args}"; - } - - private static string FormatArguments(TestContext.TestAdapter test) - { - if (test.Arguments == null || !test.Arguments.Any()) return ""; - return $"[{string.Join(',', test.Arguments)}]"; - } - } - - public class LogFile - { - private readonly DateTime now; - private string name; - private readonly string ext; - private readonly string filepath; - - public LogFile(DateTime now, string name, string ext = "log") - { - this.now = now; - this.name = name; - this.ext = ext; - - filepath = Path.Join( - LogConfig.LogRoot, - $"{now.Year}-{Pad(now.Month)}", - Pad(now.Day)); - - Directory.CreateDirectory(filepath); - - GenerateFilename(); - } - - public string FullFilename { get; private set; } = string.Empty; - public string FilenameWithoutPath { get; private set; } = string.Empty; - - public void Write(string message) - { - WriteRaw($"{GetTimestamp()} {message}"); - } - - public void WriteRaw(string message) - { - try - { - File.AppendAllLines(FullFilename, new[] { message }); - } - catch (Exception ex) - { - Console.WriteLine("Writing to log has failed: " + ex); - } - } - - public void ConcatToFilename(string toAdd) - { - var oldFullName = FullFilename; - - name += toAdd; - - GenerateFilename(); - - File.Move(oldFullName, FullFilename); - } - - private static string Pad(int n) - { - return n.ToString().PadLeft(2, '0'); - } - - private static string GetTimestamp() - { - return $"[{DateTime.UtcNow.ToString("u")}]"; - } - - private void GenerateFilename() - { - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; - FullFilename = Path.Combine(filepath, FilenameWithoutPath); - } - } -} diff --git a/CodexDistTestCore/Utils.cs b/CodexDistTestCore/Utils.cs index e0149a1..ae3e761 100644 --- a/CodexDistTestCore/Utils.cs +++ b/CodexDistTestCore/Utils.cs @@ -2,15 +2,6 @@ { public static class Utils { - public static void Sleep(TimeSpan span) - { - Thread.Sleep(span); - } - public static T Wait(Task task) - { - task.Wait(); - return task.Result; - } } } diff --git a/DistTestCore/ByteSize.cs b/DistTestCore/ByteSize.cs new file mode 100644 index 0000000..4f13da8 --- /dev/null +++ b/DistTestCore/ByteSize.cs @@ -0,0 +1,57 @@ +namespace DistTestCore +{ + public class ByteSize + { + public ByteSize(long sizeInBytes) + { + SizeInBytes = sizeInBytes; + } + + public long SizeInBytes { get; } + } + + public static class IntExtensions + { + private const long Kilo = 1024; + + public static ByteSize KB(this long i) + { + return new ByteSize(i * Kilo); + } + + public static ByteSize MB(this long i) + { + return (i * Kilo).KB(); + } + + public static ByteSize GB(this long i) + { + return (i * Kilo).MB(); + } + + public static ByteSize TB(this long i) + { + return (i * Kilo).GB(); + } + + public static ByteSize KB(this int i) + { + return Convert.ToInt64(i).KB(); + } + + public static ByteSize MB(this int i) + { + return Convert.ToInt64(i).MB(); + } + + public static ByteSize GB(this int i) + { + return Convert.ToInt64(i).GB(); + } + + public static ByteSize TB(this int i) + { + return Convert.ToInt64(i).TB(); + } + } +} diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs new file mode 100644 index 0000000..a7c826e --- /dev/null +++ b/DistTestCore/Codex/CodexAccess.cs @@ -0,0 +1,43 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexAccess + { + private readonly RunningContainer runningContainer; + + public CodexAccess(RunningContainer runningContainer) + { + this.runningContainer = runningContainer; + } + + public CodexDebugResponse GetDebugInfo() + { + var response = Http().HttpGetJson("debug/info"); + //Log($"Got DebugInfo with id: '{response.id}'."); + return response; + } + + private Http Http() + { + var ip = runningContainer.Pod.Cluster.GetIp(); + var port = runningContainer.ServicePorts[0].Number; + return new Http(ip, port, baseUrl: "/api/codex/v1"); + } + } + + public class CodexDebugResponse + { + public string id { get; set; } = string.Empty; + public string[] addrs { get; set; } = new string[0]; + public string repo { get; set; } = string.Empty; + public string spr { get; set; } = string.Empty; + public CodexDebugVersionResponse codex { get; set; } = new(); + } + + public class CodexDebugVersionResponse + { + public string version { get; set; } = string.Empty; + public string revision { get; set; } = string.Empty; + } +} diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs new file mode 100644 index 0000000..9c7b87d --- /dev/null +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -0,0 +1,35 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/nim-codex:sha-b204837"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + AddExposedPortAndVar("API_PORT"); + AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}"); + AddInternalPortAndVar("DISC_PORT"); + + var listenPort = AddInternalPort(); + AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); + + if (config.LogLevel != null) + { + AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); + } + if (config.StorageQuota != null) + { + AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!); + } + if (config.MetricsEnabled) + { + AddEnvVar("METRICS_ADDR", "0.0.0.0"); + AddInternalPortAndVar("METRICS_PORT"); + } + } + } +} diff --git a/DistTestCore/Codex/CodexLogLevel.cs b/DistTestCore/Codex/CodexLogLevel.cs new file mode 100644 index 0000000..cde0eb7 --- /dev/null +++ b/DistTestCore/Codex/CodexLogLevel.cs @@ -0,0 +1,11 @@ +namespace DistTestCore.Codex +{ + public enum CodexLogLevel + { + Trace, + Debug, + Info, + Warn, + Error + } +} diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs new file mode 100644 index 0000000..fdbffcf --- /dev/null +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -0,0 +1,12 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexStartupConfig + { + public Location Location { get; set; } + public CodexLogLevel? LogLevel { get; set; } + public ByteSize? StorageQuota { get; set; } + public bool MetricsEnabled { get; set; } + } +} diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj new file mode 100644 index 0000000..ae8e964 --- /dev/null +++ b/DistTestCore/DistTestCore.csproj @@ -0,0 +1,20 @@ + + + + net6.0 + DistTestCore + enable + enable + + + + + + + + + + + + + diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs new file mode 100644 index 0000000..365920b --- /dev/null +++ b/DistTestCore/Http.cs @@ -0,0 +1,100 @@ +using Newtonsoft.Json; +using NUnit.Framework; +using System.Net.Http.Headers; + +namespace DistTestCore +{ + public class Http + { + private readonly string ip; + private readonly int port; + private readonly string baseUrl; + + public Http(string ip, int port, string baseUrl) + { + this.ip = ip; + this.port = port; + this.baseUrl = baseUrl; + + if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; + if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; + } + + public string HttpGetString(string route) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + var result = Utils.Wait(client.GetAsync(url)); + return Utils.Wait(result.Content.ReadAsStringAsync()); + }); + } + + public T HttpGetJson(string route) + { + return JsonConvert.DeserializeObject(HttpGetString(route))!; + } + + public string HttpPostStream(string route, Stream stream) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + + var content = new StreamContent(stream); + content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + var response = Utils.Wait(client.PostAsync(url, content)); + + return Utils.Wait(response.Content.ReadAsStringAsync()); + }); + } + + public Stream HttpGetStream(string route) + { + return Retry(() => + { + var client = GetClient(); + var url = GetUrl() + route; + + return Utils.Wait(client.GetStreamAsync(url)); + }); + } + + private string GetUrl() + { + return $"http://{ip}:{port}{baseUrl}"; + } + + private static T Retry(Func operation) + { + var retryCounter = 0; + + while (true) + { + try + { + return operation(); + } + catch (Exception exception) + { + Timing.HttpCallRetryDelay(); + retryCounter++; + if (retryCounter > Timing.HttpCallRetryCount()) + { + Assert.Fail(exception.Message); + throw; + } + } + } + } + + private static HttpClient GetClient() + { + var client = new HttpClient(); + client.Timeout = Timing.HttpCallTimeout(); + return client; + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs new file mode 100644 index 0000000..e3d2186 --- /dev/null +++ b/DistTestCore/TestLifecycle.cs @@ -0,0 +1,31 @@ +using DistTestCore.Codex; +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class TestLifecycle + { + private readonly WorkflowCreator workflowCreator = new WorkflowCreator(); + + public void SetUpCodexNodes() + { + var config = new CodexStartupConfig() + { + StorageQuota = 10.MB(), + Location = Location.Unspecified, + LogLevel = CodexLogLevel.Error, + MetricsEnabled = false, + }; + + var workflow = workflowCreator.CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.Add(config); + var containers = workflow.Start(3, new CodexContainerRecipe(), startupConfig); + + foreach (var c in containers.Containers) + { + var access = new CodexAccess(c); + } + } + } +} diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs new file mode 100644 index 0000000..db8420e --- /dev/null +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -0,0 +1,42 @@ +namespace KubernetesWorkflow +{ + public class ContainerRecipe + { + public ContainerRecipe(string name, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) + { + Name = name; + Image = image; + ExposedPorts = exposedPorts; + InternalPorts = internalPorts; + EnvVars = envVars; + } + + public string Name { get; } + public string Image { get; } + public Port[] ExposedPorts { get; } + public Port[] InternalPorts { get; } + public EnvVar[] EnvVars { get; } + } + + public class Port + { + public Port(int number) + { + Number = number; + } + + public int Number { get; } + } + + public class EnvVar + { + public EnvVar(string name, string value) + { + Name = name; + Value = value; + } + + public string Name { get; } + public string Value { get; } + } +} diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs new file mode 100644 index 0000000..d4dfb0a --- /dev/null +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -0,0 +1,60 @@ +namespace KubernetesWorkflow +{ + public abstract class ContainerRecipeFactory + { + private readonly List exposedPorts = new List(); + private readonly List internalPorts = new List(); + private readonly List envVars = new List(); + private RecipeComponentFactory factory = null!; + + public ContainerRecipe CreateRecipe(int containerNumber, RecipeComponentFactory factory, StartupConfig config) + { + this.factory = factory; + ContainerNumber = containerNumber; + + Initialize(config); + + var name = $"ctnr{containerNumber}"; + + return new ContainerRecipe(name, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + } + + protected abstract string Image { get; } + protected int ContainerNumber { get; private set; } = 0; + protected abstract void Initialize(StartupConfig config); + + protected Port AddExposedPort() + { + var p = factory.CreatePort(); + exposedPorts.Add(p); + return p; + } + + protected Port AddInternalPort() + { + var p = factory.CreatePort(); + internalPorts.Add(p); + return p; + } + + protected void AddExposedPortAndVar(string name) + { + AddEnvVar(name, AddExposedPort()); + } + + protected void AddInternalPortAndVar(string name) + { + AddEnvVar(name, AddInternalPort()); + } + + protected void AddEnvVar(string name, string value) + { + envVars.Add(factory.CreateEnvVar(name, value)); + } + + protected void AddEnvVar(string name, Port value) + { + envVars.Add(factory.CreateEnvVar(name, value.Number)); + } + } +} diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs new file mode 100644 index 0000000..d325a75 --- /dev/null +++ b/KubernetesWorkflow/K8sCluster.cs @@ -0,0 +1,40 @@ +using k8s; + +namespace KubernetesWorkflow +{ + public class K8sCluster + { + public const string K8sNamespace = "codex-test-namespace"; + private const string KubeConfigFile = "C:\\kube\\config"; + private readonly Dictionary K8sNodeLocationMap = new Dictionary + { + { Location.BensLaptop, "worker01" }, + { Location.BensOldGamingMachine, "worker02" }, + }; + + private KubernetesClientConfiguration? config; + + public KubernetesClientConfiguration GetK8sClientConfig() + { + if (config != null) return config; + //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); + config = KubernetesClientConfiguration.BuildDefaultConfig(); + return config; + } + + public string GetIp() + { + var c = GetK8sClientConfig(); + + var host = c.Host.Replace("https://", ""); + + return host.Substring(0, host.IndexOf(':')); + } + + public string GetNodeLabelForLocation(Location location) + { + if (location == Location.Unspecified) return string.Empty; + return K8sNodeLocationMap[location]; + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs new file mode 100644 index 0000000..3f6bef6 --- /dev/null +++ b/KubernetesWorkflow/K8sController.cs @@ -0,0 +1,25 @@ +namespace KubernetesWorkflow +{ + public class K8sController + { + private readonly K8sCluster cluster; + + public K8sController(K8sCluster cluster) + { + this.cluster = cluster; + } + + public RunningPod BringOnline(ContainerRecipe[] containerRecipes) + { + // Ensure namespace + // create deployment + // create service if necessary + // wait until deployment online + // fetch pod info + + // for each container, there is now an array of service ports available. + + return null!; + } + } +} diff --git a/KubernetesWorkflow/KubernetesWorkflow.csproj b/KubernetesWorkflow/KubernetesWorkflow.csproj new file mode 100644 index 0000000..557b45c --- /dev/null +++ b/KubernetesWorkflow/KubernetesWorkflow.csproj @@ -0,0 +1,14 @@ + + + + net6.0 + KubernetesWorkflow + enable + enable + + + + + + + diff --git a/KubernetesWorkflow/Location.cs b/KubernetesWorkflow/Location.cs new file mode 100644 index 0000000..3b01284 --- /dev/null +++ b/KubernetesWorkflow/Location.cs @@ -0,0 +1,9 @@ +namespace KubernetesWorkflow +{ + public enum Location + { + Unspecified, + BensLaptop, + BensOldGamingMachine + } +} diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs new file mode 100644 index 0000000..6a40cc9 --- /dev/null +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -0,0 +1,24 @@ +using System.Globalization; + +namespace KubernetesWorkflow +{ + public class RecipeComponentFactory + { + private NumberSource portNumberSource = new NumberSource(8080); + + public Port CreatePort() + { + return new Port(portNumberSource.GetNextNumber()); + } + + public EnvVar CreateEnvVar(string name, int value) + { + return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture)); + } + + public EnvVar CreateEnvVar(string name, string value) + { + return new EnvVar(name, value); + } + } +} diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs new file mode 100644 index 0000000..49fc65f --- /dev/null +++ b/KubernetesWorkflow/RunningContainers.cs @@ -0,0 +1,30 @@ +namespace KubernetesWorkflow +{ + public class RunningContainers + { + public RunningContainers(StartupConfig startupConfig, RunningPod runningPod, RunningContainer[] containers) + { + StartupConfig = startupConfig; + RunningPod = runningPod; + Containers = containers; + } + + public StartupConfig StartupConfig { get; } + public RunningPod RunningPod { get; } + public RunningContainer[] Containers { get; } + } + + public class RunningContainer + { + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts) + { + Pod = pod; + Recipe = recipe; + ServicePorts = servicePorts; + } + + public RunningPod Pod { get; } + public ContainerRecipe Recipe { get; } + public Port[] ServicePorts { get; } + } +} diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs new file mode 100644 index 0000000..63378e2 --- /dev/null +++ b/KubernetesWorkflow/RunningPod.cs @@ -0,0 +1,24 @@ +namespace KubernetesWorkflow +{ + public class RunningPod + { + private readonly Dictionary servicePortMap; + + public RunningPod(K8sCluster cluster, string name, string ip, Dictionary servicePortMap) + { + Cluster = cluster; + Name = name; + Ip = ip; + this.servicePortMap = servicePortMap; + } + + public K8sCluster Cluster { get; } + public string Name { get; } + public string Ip { get; } + + public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) + { + return servicePortMap[containerRecipe]; + } + } +} diff --git a/KubernetesWorkflow/StartupConfig.cs b/KubernetesWorkflow/StartupConfig.cs new file mode 100644 index 0000000..7c13347 --- /dev/null +++ b/KubernetesWorkflow/StartupConfig.cs @@ -0,0 +1,18 @@ +namespace KubernetesWorkflow +{ + public class StartupConfig + { + private readonly List configs = new List(); + + public void Add(object config) + { + configs.Add(config); + } + + public T Get() + { + var match = configs.Single(c => c.GetType() == typeof(T)); + return (T)match; + } + } +} diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs new file mode 100644 index 0000000..eb714f7 --- /dev/null +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -0,0 +1,40 @@ +namespace KubernetesWorkflow +{ + public class StartupWorkflow + { + private readonly NumberSource containerNumberSource; + private readonly K8sController k8SController; + private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); + + public StartupWorkflow(NumberSource containerNumberSource, K8sController k8SController) + { + this.containerNumberSource = containerNumberSource; + this.k8SController = k8SController; + } + + public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + { + var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); + + var runningPod = k8SController.BringOnline(recipes); + + return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes)); + } + + private static RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes) + { + return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r))).ToArray(); + } + + private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + { + var result = new List(); + for (var i = 0; i < numberOfContainers; i++) + { + result.Add(recipeFactory.CreateRecipe(containerNumberSource.GetNextNumber(), componentFactory, startupConfig)); + } + + return result.ToArray(); + } + } +} diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs new file mode 100644 index 0000000..25f5439 --- /dev/null +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -0,0 +1,13 @@ +namespace KubernetesWorkflow +{ + public class WorkflowCreator + { + private readonly NumberSource containerNumberSource = new NumberSource(0); + private readonly K8sController controller = new K8sController(new K8sCluster()); + + public StartupWorkflow CreateWorkflow() + { + return new StartupWorkflow(containerNumberSource, controller); + } + } +} diff --git a/CodexDistTestCore/Config/LogConfig.cs b/Logging/LogConfig.cs similarity index 72% rename from CodexDistTestCore/Config/LogConfig.cs rename to Logging/LogConfig.cs index a14470e..c15b1fe 100644 --- a/CodexDistTestCore/Config/LogConfig.cs +++ b/Logging/LogConfig.cs @@ -1,4 +1,4 @@ -namespace CodexDistTestCore.Config +namespace Logging { public class LogConfig { diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs new file mode 100644 index 0000000..a43186c --- /dev/null +++ b/Logging/LogFile.cs @@ -0,0 +1,73 @@ +namespace Logging +{ + public class LogFile + { + private readonly DateTime now; + private string name; + private readonly string ext; + private readonly string filepath; + + public LogFile(DateTime now, string name, string ext = "log") + { + this.now = now; + this.name = name; + this.ext = ext; + + filepath = Path.Join( + LogConfig.LogRoot, + $"{now.Year}-{Pad(now.Month)}", + Pad(now.Day)); + + Directory.CreateDirectory(filepath); + + GenerateFilename(); + } + + public string FullFilename { get; private set; } = string.Empty; + public string FilenameWithoutPath { get; private set; } = string.Empty; + + public void Write(string message) + { + WriteRaw($"{GetTimestamp()} {message}"); + } + + public void WriteRaw(string message) + { + try + { + File.AppendAllLines(FullFilename, new[] { message }); + } + catch (Exception ex) + { + Console.WriteLine("Writing to log has failed: " + ex); + } + } + + public void ConcatToFilename(string toAdd) + { + var oldFullName = FullFilename; + + name += toAdd; + + GenerateFilename(); + + File.Move(oldFullName, FullFilename); + } + + private static string Pad(int n) + { + return n.ToString().PadLeft(2, '0'); + } + + private static string GetTimestamp() + { + return $"[{DateTime.UtcNow.ToString("u")}]"; + } + + private void GenerateFilename() + { + FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; + FullFilename = Path.Combine(filepath, FilenameWithoutPath); + } + } +} diff --git a/Logging/Logging.csproj b/Logging/Logging.csproj new file mode 100644 index 0000000..fad5ec4 --- /dev/null +++ b/Logging/Logging.csproj @@ -0,0 +1,19 @@ + + + + net6.0 + Logging + enable + enable + + + + + + + + + + + + diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs new file mode 100644 index 0000000..83b6cb1 --- /dev/null +++ b/Logging/TestLog.cs @@ -0,0 +1,74 @@ +using NUnit.Framework; +using Utils; + +namespace Logging +{ + public class TestLog + { + private readonly NumberSource subfileNumberSource = new NumberSource(0); + private readonly LogFile file; + private readonly DateTime now; + + public TestLog() + { + now = DateTime.UtcNow; + + var name = GetTestName(); + file = new LogFile(now, name); + + Log($"Begin: {name}"); + } + + public void Log(string message) + { + file.Write(message); + } + + public void Error(string message) + { + Log($"[ERROR] {message}"); + } + + public void EndTest() + { + var result = TestContext.CurrentContext.Result; + + Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); + if (!string.IsNullOrEmpty(result.Message)) + { + Log(result.Message); + Log($"{result.StackTrace}"); + } + + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + RenameLogFile(); + } + } + + private void RenameLogFile() + { + file.ConcatToFilename("_FAILED"); + } + + public LogFile CreateSubfile(string ext = "log") + { + return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); + } + + private static string GetTestName() + { + var test = TestContext.CurrentContext.Test; + var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); + var args = FormatArguments(test); + return $"{className}.{test.MethodName}{args}"; + } + + private static string FormatArguments(TestContext.TestAdapter test) + { + if (test.Arguments == null || !test.Arguments.Any()) return ""; + return $"[{string.Join(',', test.Arguments)}]"; + } + } + +} diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index b8ab20e..a7f0c9b 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -1,7 +1,7 @@ using CodexDistTestCore; using NUnit.Framework; -namespace LongTests.BasicTests +namespace TestsLong.BasicTests { [TestFixture] public class LargeFileTests : DistTest diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index 705b4fc..c39069c 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -1,7 +1,7 @@ using CodexDistTestCore; using NUnit.Framework; -namespace LongTests.BasicTests +namespace TestsLong.BasicTests { public class TestInfraTests : DistTest { diff --git a/LongTests/LongTests.csproj b/LongTests/TestsLong.csproj similarity index 100% rename from LongTests/LongTests.csproj rename to LongTests/TestsLong.csproj diff --git a/Utils/NumberSource.cs b/Utils/NumberSource.cs new file mode 100644 index 0000000..2c7266f --- /dev/null +++ b/Utils/NumberSource.cs @@ -0,0 +1,19 @@ +namespace Utils +{ + public class NumberSource + { + private int number; + + public NumberSource(int start) + { + number = start; + } + + public int GetNextNumber() + { + var n = number; + number++; + return n; + } + } +} diff --git a/Utils/Time.cs b/Utils/Time.cs new file mode 100644 index 0000000..98c5979 --- /dev/null +++ b/Utils/Time.cs @@ -0,0 +1,16 @@ +namespace Utils +{ + public static class Time + { + public static void Sleep(TimeSpan span) + { + Thread.Sleep(span); + } + + public static T Wait(Task task) + { + task.Wait(); + return task.Result; + } + } +} diff --git a/Utils/Utils.csproj b/Utils/Utils.csproj new file mode 100644 index 0000000..42c2303 --- /dev/null +++ b/Utils/Utils.csproj @@ -0,0 +1,10 @@ + + + + net6.0 + Utils + enable + enable + + + diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 9a7a7ad..43b21e9 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -3,12 +3,18 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.4.33213.308 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{57F57B85-A537-4D3A-B7AE-B72C66B74AAB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj", "{57F57B85-A537-4D3A-B7AE-B72C66B74AAB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LongTests", "LongTests\LongTests.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\TestsLong.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -27,6 +33,18 @@ Global {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.Build.0 = Debug|Any CPU {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.ActiveCfg = Release|Any CPU {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.Build.0 = Release|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.Build.0 = Release|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {359123AA-3D9B-4442-80F4-19E32E3EC9EA}.Release|Any CPU.Build.0 = Release|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 2bcf5127374ca29ec4ecc3cba4b23990410f6cd9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 15:11:36 +0200 Subject: [PATCH 02/21] Implements KubernetesWorkflow assembly. --- CodexDistTestCore/K8sOperations.cs | 38 -- KubernetesWorkflow/ContainerRecipe.cs | 7 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 4 +- KubernetesWorkflow/K8sCluster.cs | 11 + KubernetesWorkflow/K8sController.cs | 405 ++++++++++++++++++- KubernetesWorkflow/KnownK8sPods.cs | 17 + KubernetesWorkflow/KubernetesWorkflow.csproj | 4 + KubernetesWorkflow/RecipeComponentFactory.cs | 1 + KubernetesWorkflow/StartupWorkflow.cs | 49 ++- KubernetesWorkflow/WorkflowCreator.cs | 14 +- KubernetesWorkflow/WorkflowNumberSource.cs | 28 ++ cs-codex-dist-testing.sln | 6 + 12 files changed, 516 insertions(+), 68 deletions(-) create mode 100644 KubernetesWorkflow/KnownK8sPods.cs create mode 100644 KubernetesWorkflow/WorkflowNumberSource.cs diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 580829e..73f04b0 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -341,44 +341,6 @@ namespace CodexDistTestCore #endregion - #region Namespace management - - private void EnsureTestNamespace() - { - if (IsTestNamespaceOnline()) return; - - var namespaceSpec = new V1Namespace - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = K8sNamespace, - Labels = new Dictionary { { "name", K8sNamespace } } - } - }; - client.CreateNamespace(namespaceSpec); - } - - private void DeleteNamespace() - { - if (IsTestNamespaceOnline()) - { - client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); - } - } - - private string K8sNamespace - { - get { return K8sCluster.K8sNamespace; } - } - - #endregion - - private bool IsTestNamespaceOnline() - { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); - } - private class CommandRunner { private readonly Kubernetes client; diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index db8420e..7e8d90e 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -2,16 +2,17 @@ { public class ContainerRecipe { - public ContainerRecipe(string name, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) + public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) { - Name = name; + Number = number; Image = image; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; } - public string Name { get; } + public string Name { get { return $"ctnr{Number}"; } } + public int Number { get; } public string Image { get; } public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index d4dfb0a..fc06f33 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -14,9 +14,7 @@ Initialize(config); - var name = $"ctnr{containerNumber}"; - - return new ContainerRecipe(name, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + return new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); } protected abstract string Image { get; } diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index d325a75..7d6e8eb 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -36,5 +36,16 @@ namespace KubernetesWorkflow if (location == Location.Unspecified) return string.Empty; return K8sNodeLocationMap[location]; } + + // make configurable from test env! + public TimeSpan K8sOperationTimeout() + { + return TimeSpan.FromMinutes(5); + } + + public TimeSpan WaitForK8sServiceDelay() + { + return TimeSpan.FromSeconds(5); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 3f6bef6..4049607 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -1,25 +1,410 @@ -namespace KubernetesWorkflow +using k8s; +using k8s.Models; + +namespace KubernetesWorkflow { public class K8sController { private readonly K8sCluster cluster; + private readonly KnownK8sPods knownPods; + private readonly WorkflowNumberSource workflowNumberSource; + private readonly Kubernetes client; - public K8sController(K8sCluster cluster) + public K8sController(K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource) { this.cluster = cluster; + this.knownPods = knownPods; + this.workflowNumberSource = workflowNumberSource; + + client = new Kubernetes(cluster.GetK8sClientConfig()); } - public RunningPod BringOnline(ContainerRecipe[] containerRecipes) + public void Dispose() { - // Ensure namespace - // create deployment - // create service if necessary - // wait until deployment online - // fetch pod info + client.Dispose(); + } + + public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location) + { + EnsureTestNamespace(); - // for each container, there is now an array of service ports available. + CreateDeployment(containerRecipes, location); + var servicePortsMap = CreateService(containerRecipes); + var (podName, podIp) = FetchNewPod(); - return null!; + return new RunningPod(cluster, podName, podIp, servicePortsMap); + } + + public void DeleteAllResources() + { + DeleteNamespace(); + + WaitUntilNamespaceDeleted(); + } + + #region Namespace management + + private void EnsureTestNamespace() + { + if (IsTestNamespaceOnline()) return; + + var namespaceSpec = new V1Namespace + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = K8sNamespace, + Labels = new Dictionary { { "name", K8sNamespace } } + } + }; + client.CreateNamespace(namespaceSpec); + WaitUntilNamespaceCreated(); + } + + private void DeleteNamespace() + { + if (IsTestNamespaceOnline()) + { + client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); + } + } + + private string K8sNamespace + { + get { return K8sCluster.K8sNamespace; } + } + + private bool IsTestNamespaceOnline() + { + return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); + } + + #endregion + + #region Deployment management + + private void CreateDeployment(ContainerRecipe[] containerRecipes, Location location) + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = CreateDeploymentMetadata(), + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = GetSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = GetSelector() + }, + Spec = new V1PodSpec + { + NodeSelector = CreateNodeSelector(location), + Containers = CreateDeploymentContainers(containerRecipes) + } + } + } + }; + + client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + WaitUntilDeploymentCreated(deploymentSpec); + } + + private IDictionary CreateNodeSelector(Location location) + { + if (location == Location.Unspecified) return new Dictionary(); + + return new Dictionary + { + { "codex-test-location", cluster.GetNodeLabelForLocation(location) } + }; + } + + private IDictionary GetSelector() + { + return new Dictionary { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } }; + } + + private V1ObjectMeta CreateDeploymentMetadata() + { + return new V1ObjectMeta + { + Name = "deploy-" + workflowNumberSource.WorkflowNumber, + NamespaceProperty = K8sCluster.K8sNamespace + }; + } + + private List CreateDeploymentContainers(ContainerRecipe[] containerRecipes) + { + return containerRecipes.Select(r => CreateDeploymentContainer(r)).ToList(); + } + + private V1Container CreateDeploymentContainer(ContainerRecipe recipe) + { + return new V1Container + { + Name = recipe.Name, + Image = recipe.Image, + Ports = CreateContainerPorts(recipe), + Env = CreateEnv(recipe) + }; + } + + private List CreateEnv(ContainerRecipe recipe) + { + return recipe.EnvVars.Select(CreateEnvVar).ToList(); + } + + private V1EnvVar CreateEnvVar(EnvVar envVar) + { + return new V1EnvVar + { + Name = envVar.Name, + Value = envVar.Value, + }; + } + + private List CreateContainerPorts(ContainerRecipe recipe) + { + var exposedPorts = recipe.ExposedPorts.Select(p => CreateContainerPort(recipe, p)); + var internalPorts = recipe.InternalPorts.Select(p => CreateContainerPort(recipe, p)); + return exposedPorts.Concat(internalPorts).ToList(); + } + + private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port) + { + return new V1ContainerPort + { + Name = GetNameForPort(recipe, port), + ContainerPort = port.Number + }; + } + + private string GetNameForPort(ContainerRecipe recipe, Port port) + { + return $"P{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}"; + } + + + //private void DeleteDeployment(CodexNodeGroup group) + //{ + // if (group.Deployment == null) return; + // client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace); + // group.Deployment = null; + //} + + //private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) + //{ + // client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); + //} + + //private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec) + //{ + // client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); + //} + + //private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) + //{ + // client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); + //} + + #endregion + + #region Service management + + private Dictionary CreateService(ContainerRecipe[] containerRecipes) + { + var result = new Dictionary(); + + var ports = CreateServicePorts(result, containerRecipes); + + if (!ports.Any()) + { + // None of these container-recipes wish to expose anything via a serice port. + // So, we don't have to create a service. + return result; + } + + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = CreateServiceMetadata(), + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = GetSelector(), + Ports = ports + } + }; + + client.CreateNamespacedService(serviceSpec, K8sNamespace); + return result; + } + + private V1ObjectMeta CreateServiceMetadata() + { + return new V1ObjectMeta + { + Name = "deploy-" + workflowNumberSource.WorkflowNumber, + NamespaceProperty = K8sCluster.K8sNamespace + }; + } + + private List CreateServicePorts(Dictionary servicePorts, ContainerRecipe[] recipes) + { + var result = new List(); + foreach (var recipe in recipes) + { + result.AddRange(CreateServicePorts(servicePorts, recipe)); + } + return result; + } + + private List CreateServicePorts(Dictionary servicePorts, ContainerRecipe recipe) + { + var result = new List(); + var usedPorts = new List(); + foreach (var port in recipe.ExposedPorts) + { + var servicePort = workflowNumberSource.GetServicePort(); + usedPorts.Add(new Port(servicePort)); + + result.Add(new V1ServicePort + { + Name = GetNameForPort(recipe, port), + Protocol = "TCP", + Port = port.Number, + TargetPort = GetNameForPort(recipe, port), + NodePort = servicePort + }); + } + + servicePorts.Add(recipe, usedPorts.ToArray()); + return result; + } + + //private void DeleteService(CodexNodeGroup online) + //{ + // if (online.Service == null) return; + // client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); + // online.Service = null; + //} + + //private void CreatePrometheusService(K8sPrometheusSpecs spec) + //{ + // client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); + //} + + //private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec) + //{ + // client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace); + //} + + #endregion + + #region Waiting + + //private void WaitUntilOnline(CodexNodeGroup online) + //{ + // WaitUntil(() => + // { + // online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + // return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; + // }); + //} + + //private void WaitUntilOffline(string deploymentName) + //{ + // WaitUntil(() => + // { + // var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + // return deployment == null || deployment.Status.AvailableReplicas == 0; + // }); + //} + + //private void WaitUntilZeroPods() + //{ + // WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); + //} + + private void WaitUntilNamespaceCreated() + { + WaitUntil(() => IsTestNamespaceOnline()); + } + + private void WaitUntilNamespaceDeleted() + { + WaitUntil(() => !IsTestNamespaceOnline()); + } + + //private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) + //{ + // WaitUntilDeploymentOnline(spec.GetDeploymentName()); + //} + + //private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) + //{ + // WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); + //} + + //private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) + //{ + // WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); + //} + + private void WaitUntilDeploymentCreated(V1Deployment deploymentSpec) + { + WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); + } + + private void WaitUntilDeploymentOnline(string deploymentName) + { + WaitUntil(() => + { + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; + }); + } + + private void WaitUntil(Func predicate) + { + var start = DateTime.UtcNow; + var state = predicate(); + while (!state) + { + if (DateTime.UtcNow - start > cluster.K8sOperationTimeout()) + { + throw new TimeoutException("K8s operation timed out."); + } + + cluster.WaitForK8sServiceDelay(); + state = predicate(); + } + } + + #endregion + + private (string, string) FetchNewPod() + { + var pods = client.ListNamespacedPod(K8sNamespace).Items; + + var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); + if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure."); + + var newPod = newPods.Single(); + var name = newPod.Name(); + var ip = newPod.Status.PodIP; + + if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure."); + if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure."); + + knownPods.Add(name); + return (name, ip); } } } diff --git a/KubernetesWorkflow/KnownK8sPods.cs b/KubernetesWorkflow/KnownK8sPods.cs new file mode 100644 index 0000000..6d80eb6 --- /dev/null +++ b/KubernetesWorkflow/KnownK8sPods.cs @@ -0,0 +1,17 @@ +namespace KubernetesWorkflow +{ + public class KnownK8sPods + { + private readonly List knownActivePodNames = new List(); + + public bool Contains(string name) + { + return knownActivePodNames.Contains(name); + } + + public void Add(string name) + { + knownActivePodNames.Add(name); + } + } +} diff --git a/KubernetesWorkflow/KubernetesWorkflow.csproj b/KubernetesWorkflow/KubernetesWorkflow.csproj index 557b45c..655a8fd 100644 --- a/KubernetesWorkflow/KubernetesWorkflow.csproj +++ b/KubernetesWorkflow/KubernetesWorkflow.csproj @@ -11,4 +11,8 @@ + + + + diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs index 6a40cc9..f99f345 100644 --- a/KubernetesWorkflow/RecipeComponentFactory.cs +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -1,4 +1,5 @@ using System.Globalization; +using Utils; namespace KubernetesWorkflow { diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index eb714f7..8486590 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -2,23 +2,36 @@ { public class StartupWorkflow { - private readonly NumberSource containerNumberSource; - private readonly K8sController k8SController; + private readonly WorkflowNumberSource numberSource; + private readonly K8sCluster cluster; + private readonly KnownK8sPods knownK8SPods; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - public StartupWorkflow(NumberSource containerNumberSource, K8sController k8SController) + internal StartupWorkflow(WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods) { - this.containerNumberSource = containerNumberSource; - this.k8SController = k8SController; + this.numberSource = numberSource; + this.cluster = cluster; + this.knownK8SPods = knownK8SPods; } - public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { - var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); + return K8s(controller => + { + var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig); - var runningPod = k8SController.BringOnline(recipes); + var runningPod = controller.BringOnline(recipes, location); - return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes)); + return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes)); + }); + } + + public void DeleteAllResources() + { + K8s(controller => + { + controller.DeleteAllResources(); + }); } private static RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes) @@ -31,10 +44,26 @@ var result = new List(); for (var i = 0; i < numberOfContainers; i++) { - result.Add(recipeFactory.CreateRecipe(containerNumberSource.GetNextNumber(), componentFactory, startupConfig)); + result.Add(recipeFactory.CreateRecipe(numberSource.GetContainerNumber(), componentFactory, startupConfig)); } return result.ToArray(); } + + private void K8s(Action action) + { + var controller = new K8sController(cluster, knownK8SPods, numberSource); + action(controller); + controller.Dispose(); + } + + private T K8s(Func action) + { + var controller = new K8sController(cluster, knownK8SPods, numberSource); + var result = action(controller); + controller.Dispose(); + return result; + } + } } diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 25f5439..450bdd6 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -1,13 +1,19 @@ -namespace KubernetesWorkflow +using Utils; + +namespace KubernetesWorkflow { public class WorkflowCreator { - private readonly NumberSource containerNumberSource = new NumberSource(0); - private readonly K8sController controller = new K8sController(new K8sCluster()); + private readonly NumberSource numberSource = new NumberSource(0); + private readonly NumberSource servicePortNumberSource = new NumberSource(30001); + private readonly K8sCluster cluster = new K8sCluster(); + private readonly KnownK8sPods knownPods = new KnownK8sPods(); public StartupWorkflow CreateWorkflow() { - return new StartupWorkflow(containerNumberSource, controller); + var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), servicePortNumberSource); + + return new StartupWorkflow(workflowNumberSource, cluster, knownPods); } } } diff --git a/KubernetesWorkflow/WorkflowNumberSource.cs b/KubernetesWorkflow/WorkflowNumberSource.cs new file mode 100644 index 0000000..018b97b --- /dev/null +++ b/KubernetesWorkflow/WorkflowNumberSource.cs @@ -0,0 +1,28 @@ +using Utils; + +namespace KubernetesWorkflow +{ + public class WorkflowNumberSource + { + private readonly NumberSource containerNumberSource = new NumberSource(0); + private readonly NumberSource servicePortNumberSource; + + public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource) + { + WorkflowNumber = workflowNumber; + this.servicePortNumberSource = servicePortNumberSource; + } + + public int WorkflowNumber { get; } + + public int GetContainerNumber() + { + return containerNumberSource.GetNextNumber(); + } + + public int GetServicePort() + { + return servicePortNumberSource.GetNextNumber(); + } + } +} diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 43b21e9..de9f58f 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -15,6 +15,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KubernetesWorkflow", "Kuber EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -45,6 +47,10 @@ Global {957DE3B8-9571-450A-8609-B267DCA8727C}.Debug|Any CPU.Build.0 = Debug|Any CPU {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.ActiveCfg = Release|Any CPU {957DE3B8-9571-450A-8609-B267DCA8727C}.Release|Any CPU.Build.0 = Release|Any CPU + {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 7c8a278cd99a5982ab4cc6df033d5649c2fd8a87 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 15:22:09 +0200 Subject: [PATCH 03/21] Cleanup and make kubernetesworkflow configurable --- KubernetesWorkflow/Configuration.cs | 32 ++++++++++ KubernetesWorkflow/K8sCluster.cs | 34 ++++++----- KubernetesWorkflow/K8sController.cs | 84 +-------------------------- KubernetesWorkflow/WorkflowCreator.cs | 7 ++- 4 files changed, 61 insertions(+), 96 deletions(-) create mode 100644 KubernetesWorkflow/Configuration.cs diff --git a/KubernetesWorkflow/Configuration.cs b/KubernetesWorkflow/Configuration.cs new file mode 100644 index 0000000..b5a4779 --- /dev/null +++ b/KubernetesWorkflow/Configuration.cs @@ -0,0 +1,32 @@ +namespace KubernetesWorkflow +{ + public class Configuration + { + public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) + { + K8sNamespace = k8sNamespace; + KubeConfigFile = kubeConfigFile; + OperationTimeout = operationTimeout; + RetryDelay = retryDelay; + LocationMap = locationMap; + } + + public string K8sNamespace { get; } + public string? KubeConfigFile { get; } + public TimeSpan OperationTimeout { get; } + public TimeSpan RetryDelay { get; } + public ConfigurationLocationEntry[] LocationMap { get; } + } + + public class ConfigurationLocationEntry + { + public ConfigurationLocationEntry(Location location, string workerName) + { + Location = location; + WorkerName = workerName; + } + + public Location Location { get; } + public string WorkerName { get; } + } +} diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index 7d6e8eb..c823a09 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -4,21 +4,28 @@ namespace KubernetesWorkflow { public class K8sCluster { - public const string K8sNamespace = "codex-test-namespace"; - private const string KubeConfigFile = "C:\\kube\\config"; - private readonly Dictionary K8sNodeLocationMap = new Dictionary - { - { Location.BensLaptop, "worker01" }, - { Location.BensOldGamingMachine, "worker02" }, - }; - private KubernetesClientConfiguration? config; + public K8sCluster(Configuration configuration) + { + Configuration = configuration; + } + + public Configuration Configuration { get; } + public KubernetesClientConfiguration GetK8sClientConfig() { if (config != null) return config; - //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); - config = KubernetesClientConfiguration.BuildDefaultConfig(); + + if (Configuration.KubeConfigFile != null) + { + config = KubernetesClientConfiguration.BuildConfigFromConfigFile(Configuration.KubeConfigFile); + } + else + { + config = KubernetesClientConfiguration.BuildDefaultConfig(); + } + return config; } @@ -34,18 +41,17 @@ namespace KubernetesWorkflow public string GetNodeLabelForLocation(Location location) { if (location == Location.Unspecified) return string.Empty; - return K8sNodeLocationMap[location]; + return Configuration.LocationMap.Single(l => l.Location == location).WorkerName; } - // make configurable from test env! public TimeSpan K8sOperationTimeout() { - return TimeSpan.FromMinutes(5); + return Configuration.OperationTimeout; } public TimeSpan WaitForK8sServiceDelay() { - return TimeSpan.FromSeconds(5); + return Configuration.RetryDelay; } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 4049607..df68f18 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -71,7 +71,7 @@ namespace KubernetesWorkflow private string K8sNamespace { - get { return K8sCluster.K8sNamespace; } + get { return cluster.Configuration.K8sNamespace; } } private bool IsTestNamespaceOnline() @@ -135,7 +135,7 @@ namespace KubernetesWorkflow return new V1ObjectMeta { Name = "deploy-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sCluster.K8sNamespace + NamespaceProperty = K8sNamespace }; } @@ -190,29 +190,6 @@ namespace KubernetesWorkflow return $"P{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}"; } - - //private void DeleteDeployment(CodexNodeGroup group) - //{ - // if (group.Deployment == null) return; - // client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace); - // group.Deployment = null; - //} - - //private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) - //{ - // client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); - //} - - //private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec) - //{ - // client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); - //} - - //private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) - //{ - // client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); - //} - #endregion #region Service management @@ -251,7 +228,7 @@ namespace KubernetesWorkflow return new V1ObjectMeta { Name = "deploy-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sCluster.K8sNamespace + NamespaceProperty = K8sNamespace }; } @@ -288,50 +265,10 @@ namespace KubernetesWorkflow return result; } - //private void DeleteService(CodexNodeGroup online) - //{ - // if (online.Service == null) return; - // client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); - // online.Service = null; - //} - - //private void CreatePrometheusService(K8sPrometheusSpecs spec) - //{ - // client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); - //} - - //private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec) - //{ - // client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace); - //} - #endregion #region Waiting - //private void WaitUntilOnline(CodexNodeGroup online) - //{ - // WaitUntil(() => - // { - // online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - // return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; - // }); - //} - - //private void WaitUntilOffline(string deploymentName) - //{ - // WaitUntil(() => - // { - // var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - // return deployment == null || deployment.Status.AvailableReplicas == 0; - // }); - //} - - //private void WaitUntilZeroPods() - //{ - // WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); - //} - private void WaitUntilNamespaceCreated() { WaitUntil(() => IsTestNamespaceOnline()); @@ -342,21 +279,6 @@ namespace KubernetesWorkflow WaitUntil(() => !IsTestNamespaceOnline()); } - //private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) - //{ - // WaitUntilDeploymentOnline(spec.GetDeploymentName()); - //} - - //private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) - //{ - // WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); - //} - - //private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) - //{ - // WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); - //} - private void WaitUntilDeploymentCreated(V1Deployment deploymentSpec) { WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 450bdd6..88af0e8 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -6,8 +6,13 @@ namespace KubernetesWorkflow { private readonly NumberSource numberSource = new NumberSource(0); private readonly NumberSource servicePortNumberSource = new NumberSource(30001); - private readonly K8sCluster cluster = new K8sCluster(); private readonly KnownK8sPods knownPods = new KnownK8sPods(); + private readonly K8sCluster cluster; + + public WorkflowCreator(Configuration configuration) + { + cluster = new K8sCluster(configuration); + } public StartupWorkflow CreateWorkflow() { From 68d089874de8b4ee8ebbaf42f956a98c3c4e72a5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 16:06:04 +0200 Subject: [PATCH 04/21] wiring up the dist-test backend --- CodexDistTestCore/Config/K8sCluster.cs | 2 +- DistTestCore/CodexSetupConfig.cs | 97 ++++++++++++++++++ DistTestCore/CodexStarter.cs | 26 +++++ DistTestCore/Configuration.cs | 32 ++++++ DistTestCore/DistTest.cs | 123 +++++++++++++++++++++++ DistTestCore/FileManager.cs | 113 +++++++++++++++++++++ DistTestCore/TestLifecycle.cs | 33 +++---- DistTestCore/Timing.cs | 132 +++++++++++++++++++++++++ Logging/LogConfig.cs | 7 +- Logging/LogFile.cs | 4 +- Logging/TestLog.cs | 9 +- 11 files changed, 550 insertions(+), 28 deletions(-) create mode 100644 DistTestCore/CodexSetupConfig.cs create mode 100644 DistTestCore/CodexStarter.cs create mode 100644 DistTestCore/Configuration.cs create mode 100644 DistTestCore/DistTest.cs create mode 100644 DistTestCore/FileManager.cs create mode 100644 DistTestCore/Timing.cs diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs index a290193..a17e6fd 100644 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ b/CodexDistTestCore/Config/K8sCluster.cs @@ -4,7 +4,7 @@ namespace CodexDistTestCore.Config { public class K8sCluster { - public const string K8sNamespace = "codex-test-namespace"; + public const string K8sNamespace = ""; private const string KubeConfigFile = "C:\\kube\\config"; private readonly Dictionary K8sNodeLocationMap = new Dictionary { diff --git a/DistTestCore/CodexSetupConfig.cs b/DistTestCore/CodexSetupConfig.cs new file mode 100644 index 0000000..1f1a632 --- /dev/null +++ b/DistTestCore/CodexSetupConfig.cs @@ -0,0 +1,97 @@ +using DistTestCore.Codex; + +namespace DistTestCore +{ + public interface ICodexSetupConfig + { + ICodexSetupConfig At(Location location); + ICodexSetupConfig WithLogLevel(CodexLogLevel level); + //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); + ICodexSetupConfig WithStorageQuota(ByteSize storageQuota); + ICodexSetupConfig EnableMetrics(); + //ICodexSetupConfig EnableMarketplace(int initialBalance); + ICodexNodeGroup BringOnline(); + } + + public enum Location + { + Unspecified, + BensLaptop, + BensOldGamingMachine, + } + + public class CodexSetupConfig : ICodexSetupConfig + { + private readonly CodexStarter starter; + + public int NumberOfNodes { get; } + public Location Location { get; private set; } + public CodexLogLevel? LogLevel { get; private set; } + //public IOnlineCodexNode? BootstrapNode { get; private set; } + public ByteSize? StorageQuota { get; private set; } + public bool MetricsEnabled { get; private set; } + //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } + + public CodexSetupConfig(CodexStarter starter, int numberOfNodes) + { + this.starter = starter; + NumberOfNodes = numberOfNodes; + Location = Location.Unspecified; + MetricsEnabled = false; + } + + public ICodexNodeGroup BringOnline() + { + return starter.BringOnline(this); + } + + public ICodexSetupConfig At(Location location) + { + Location = location; + return this; + } + + //public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node) + //{ + // BootstrapNode = node; + // return this; + //} + + public ICodexSetupConfig WithLogLevel(CodexLogLevel level) + { + LogLevel = level; + return this; + } + + public ICodexSetupConfig WithStorageQuota(ByteSize storageQuota) + { + StorageQuota = storageQuota; + return this; + } + + public ICodexSetupConfig EnableMetrics() + { + MetricsEnabled = true; + return this; + } + + //public ICodexSetupConfig EnableMarketplace(int initialBalance) + //{ + // MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); + // return this; + //} + + public string Describe() + { + var args = string.Join(',', DescribeArgs()); + return $"{NumberOfNodes} CodexNodes with [{args}]"; + } + + private IEnumerable DescribeArgs() + { + if (LogLevel != null) yield return $"LogLevel={LogLevel}"; + //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here"; + if (StorageQuota != null) yield return $"StorageQuote={StorageQuota.SizeInBytes}"; + } + } +} diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs new file mode 100644 index 0000000..f295d0a --- /dev/null +++ b/DistTestCore/CodexStarter.cs @@ -0,0 +1,26 @@ +using KubernetesWorkflow; +using Logging; + +namespace DistTestCore +{ + public class CodexStarter + { + private readonly WorkflowCreator workflowCreator; + + public CodexStarter(TestLog log, Configuration configuration) + { + workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + } + + public ICodexNodeGroup BringOnline(CodexSetupConfig codexSetupConfig) + { + + } + + public void DeleteAllResources() + { + var workflow = workflowCreator.CreateWorkflow(); + workflow.DeleteAllResources(); + } + } +} diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs new file mode 100644 index 0000000..f11437c --- /dev/null +++ b/DistTestCore/Configuration.cs @@ -0,0 +1,32 @@ +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class Configuration + { + public KubernetesWorkflow.Configuration GetK8sConfiguration() + { + return new KubernetesWorkflow.Configuration( + k8sNamespace: "codex-test-ns", + kubeConfigFile: null, + operationTimeout: Timing.K8sOperationTimeout(), + retryDelay: Timing.K8sServiceDelay(), + locationMap: new[] + { + new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"), + new ConfigurationLocationEntry(Location.BensLaptop, "worker02"), + } + ); + } + + public Logging.LogConfig GetLogConfig() + { + return new Logging.LogConfig("D:/CodexTestLogs"); + } + + public string GetFileManagerFolder() + { + return "TestDataFiles"; + } + } +} diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs new file mode 100644 index 0000000..995fffa --- /dev/null +++ b/DistTestCore/DistTest.cs @@ -0,0 +1,123 @@ +using NUnit.Framework; + +namespace DistTestCore +{ + [SetUpFixture] + public abstract class DistTest + { + private TestLifecycle lifecycle = null!; + + [OneTimeSetUp] + public void GlobalSetup() + { + // Previous test run may have been interrupted. + // Begin by cleaning everything up. + CreateNewTestLifecycle(); + + try + { + lifecycle.DeleteAllResources(); + } + catch (Exception ex) + { + GlobalTestFailure.HasFailed = true; + Error($"Global setup cleanup failed with: {ex}"); + throw; + } + Log("Global setup cleanup successful"); + } + + [SetUp] + public void SetUpDistTest() + { + if (GlobalTestFailure.HasFailed) + { + Assert.Inconclusive("Skip test: Previous test failed during clean up."); + } + else + { + CreateNewTestLifecycle(); + } + } + + [TearDown] + public void TearDownDistTest() + { + try + { + lifecycle.Log.EndTest(); + IncludeLogsAndMetricsOnTestFailure(); + lifecycle.DeleteAllResources(); + } + catch (Exception ex) + { + Error("Cleanup failed: " + ex.Message); + GlobalTestFailure.HasFailed = true; + } + } + + public TestFile GenerateTestFile(ByteSize size) + { + return lifecycle.FileManager.GenerateTestFile(size); + } + + public ICodexSetupConfig SetupCodexNodes(int numberOfNodes) + { + return new CodexSetupConfig(lifecycle.CodexStarter, numberOfNodes); + } + + private void IncludeLogsAndMetricsOnTestFailure() + { + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + if (IsDownloadingLogsAndMetricsEnabled()) + { + log.Log("Downloading all CodexNode logs and metrics because of test failure..."); + k8sManager.ForEachOnlineGroup(DownloadLogs); + k8sManager.DownloadAllMetrics(); + } + else + { + log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + } + } + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + + private void Error(string msg) + { + lifecycle.Log.Error(msg); + } + + private void CreateNewTestLifecycle() + { + lifecycle = new TestLifecycle(new Configuration()); + } + + private void DownloadLogs(CodexNodeGroup group) + { + foreach (var node in group) + { + var downloader = new PodLogDownloader(log, k8sManager); + var n = (OnlineCodexNode)node; + downloader.DownloadLog(n); + } + } + + private bool IsDownloadingLogsAndMetricsEnabled() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); + } + } + + public static class GlobalTestFailure + { + public static bool HasFailed { get; set; } = false; + } +} diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs new file mode 100644 index 0000000..10f126b --- /dev/null +++ b/DistTestCore/FileManager.cs @@ -0,0 +1,113 @@ +using Logging; +using NUnit.Framework; + +namespace DistTestCore +{ + public interface IFileManager + { + TestFile CreateEmptyTestFile(); + TestFile GenerateTestFile(ByteSize size); + void DeleteAllTestFiles(); + } + + public class FileManager : IFileManager + { + public const int ChunkSize = 1024 * 1024; + private readonly Random random = new Random(); + private readonly List activeFiles = new List(); + private readonly TestLog log; + private readonly string folder; + + public FileManager(TestLog log, Configuration configuration) + { + folder = configuration.GetFileManagerFolder(); + + if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + this.log = log; + } + + public TestFile CreateEmptyTestFile() + { + var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin")); + File.Create(result.Filename).Close(); + activeFiles.Add(result); + return result; + } + + public TestFile GenerateTestFile(ByteSize size) + { + var result = CreateEmptyTestFile(); + GenerateFileBytes(result, size); + log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); + return result; + } + + public void DeleteAllTestFiles() + { + foreach (var file in activeFiles) File.Delete(file.Filename); + activeFiles.Clear(); + } + + private void GenerateFileBytes(TestFile result, ByteSize size) + { + long bytesLeft = size.SizeInBytes; + while (bytesLeft > 0) + { + var length = Math.Min(bytesLeft, ChunkSize); + AppendRandomBytesToFile(result, length); + bytesLeft -= length; + } + } + + private void AppendRandomBytesToFile(TestFile result, long length) + { + var bytes = new byte[length]; + random.NextBytes(bytes); + using var stream = new FileStream(result.Filename, FileMode.Append); + stream.Write(bytes, 0, bytes.Length); + } + } + + public class TestFile + { + public TestFile(string filename) + { + Filename = filename; + } + + public string Filename { get; } + + public long GetFileSize() + { + var info = new FileInfo(Filename); + return info.Length; + } + + public void AssertIsEqual(TestFile? actual) + { + if (actual == null) Assert.Fail("TestFile is null."); + if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + + Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); + + using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); + using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); + + var bytesExpected = new byte[FileManager.ChunkSize]; + var bytesActual = new byte[FileManager.ChunkSize]; + + var readExpected = 0; + var readActual = 0; + + while (true) + { + readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); + readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); + + if (readExpected == 0 && readActual == 0) return; + Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); + CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); + } + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index e3d2186..6ff92f9 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,31 +1,24 @@ -using DistTestCore.Codex; -using KubernetesWorkflow; +using Logging; namespace DistTestCore { public class TestLifecycle { - private readonly WorkflowCreator workflowCreator = new WorkflowCreator(); - - public void SetUpCodexNodes() + public TestLifecycle(Configuration configuration) { - var config = new CodexStartupConfig() - { - StorageQuota = 10.MB(), - Location = Location.Unspecified, - LogLevel = CodexLogLevel.Error, - MetricsEnabled = false, - }; + Log = new TestLog(configuration.GetLogConfig()); + FileManager = new FileManager(Log, configuration); + CodexStarter = new CodexStarter(Log, configuration); + } - var workflow = workflowCreator.CreateWorkflow(); - var startupConfig = new StartupConfig(); - startupConfig.Add(config); - var containers = workflow.Start(3, new CodexContainerRecipe(), startupConfig); + public TestLog Log { get; } + public FileManager FileManager { get; } + public CodexStarter CodexStarter { get; } - foreach (var c in containers.Containers) - { - var access = new CodexAccess(c); - } + public void DeleteAllResources() + { + CodexStarter.DeleteAllResources(); + FileManager.DeleteAllTestFiles(); } } } diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs new file mode 100644 index 0000000..67653ec --- /dev/null +++ b/DistTestCore/Timing.cs @@ -0,0 +1,132 @@ +using NUnit.Framework; +using Utils; + +namespace DistTestCore +{ + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class UseLongTimeoutsAttribute : PropertyAttribute + { + public UseLongTimeoutsAttribute() + : base(Timing.UseLongTimeoutsKey) + { + } + } + + public static class Timing + { + public const string UseLongTimeoutsKey = "UseLongTimeouts"; + + public static TimeSpan HttpCallTimeout() + { + return GetTimes().HttpCallTimeout(); + } + + public static int HttpCallRetryCount() + { + return GetTimes().HttpCallRetryCount(); + } + + public static void HttpCallRetryDelay() + { + Time.Sleep(GetTimes().HttpCallRetryDelay()); + } + + public static TimeSpan K8sServiceDelay() + { + return GetTimes().WaitForK8sServiceDelay(); + } + + public static TimeSpan K8sOperationTimeout() + { + return GetTimes().K8sOperationTimeout(); + } + + public static TimeSpan WaitForMetricTimeout() + { + return GetTimes().WaitForMetricTimeout(); + } + + private static ITimeSet GetTimes() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet(); + return new DefaultTimeSet(); + } + } + + public interface ITimeSet + { + TimeSpan HttpCallTimeout(); + int HttpCallRetryCount(); + TimeSpan HttpCallRetryDelay(); + TimeSpan WaitForK8sServiceDelay(); + TimeSpan K8sOperationTimeout(); + TimeSpan WaitForMetricTimeout(); + } + + public class DefaultTimeSet : ITimeSet + { + public TimeSpan HttpCallTimeout() + { + return TimeSpan.FromSeconds(10); + } + + public int HttpCallRetryCount() + { + return 5; + } + + public TimeSpan HttpCallRetryDelay() + { + return TimeSpan.FromSeconds(3); + } + + public TimeSpan WaitForK8sServiceDelay() + { + return TimeSpan.FromSeconds(1); + } + + public TimeSpan K8sOperationTimeout() + { + return TimeSpan.FromMinutes(5); + } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromSeconds(30); + } + } + + public class LongTimeSet : ITimeSet + { + public TimeSpan HttpCallTimeout() + { + return TimeSpan.FromHours(2); + } + + public int HttpCallRetryCount() + { + return 2; + } + + public TimeSpan HttpCallRetryDelay() + { + return TimeSpan.FromMinutes(5); + } + + public TimeSpan WaitForK8sServiceDelay() + { + return TimeSpan.FromSeconds(10); + } + + public TimeSpan K8sOperationTimeout() + { + return TimeSpan.FromMinutes(15); + } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromMinutes(5); + } + } +} diff --git a/Logging/LogConfig.cs b/Logging/LogConfig.cs index c15b1fe..b7bc937 100644 --- a/Logging/LogConfig.cs +++ b/Logging/LogConfig.cs @@ -2,6 +2,11 @@ { public class LogConfig { - public const string LogRoot = "D:/CodexTestLogs"; + public LogConfig(string logRoot) + { + LogRoot = logRoot; + } + + public string LogRoot { get; } } } diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs index a43186c..3a0063b 100644 --- a/Logging/LogFile.cs +++ b/Logging/LogFile.cs @@ -7,14 +7,14 @@ private readonly string ext; private readonly string filepath; - public LogFile(DateTime now, string name, string ext = "log") + public LogFile(LogConfig config, DateTime now, string name, string ext = "log") { this.now = now; this.name = name; this.ext = ext; filepath = Path.Join( - LogConfig.LogRoot, + config.LogRoot, $"{now.Year}-{Pad(now.Month)}", Pad(now.Day)); diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 83b6cb1..7eb3979 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -8,13 +8,15 @@ namespace Logging private readonly NumberSource subfileNumberSource = new NumberSource(0); private readonly LogFile file; private readonly DateTime now; + private readonly LogConfig config; - public TestLog() + public TestLog(LogConfig config) { + this.config = config; now = DateTime.UtcNow; var name = GetTestName(); - file = new LogFile(now, name); + file = new LogFile(config, now, name); Log($"Begin: {name}"); } @@ -53,7 +55,7 @@ namespace Logging public LogFile CreateSubfile(string ext = "log") { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); + return new LogFile(config, now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); } private static string GetTestName() @@ -70,5 +72,4 @@ namespace Logging return $"[{string.Join(',', test.Arguments)}]"; } } - } From bb81d7f0377d80d05d1904fd0f924282dc5adeb1 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 12 Apr 2023 16:12:04 +0200 Subject: [PATCH 05/21] rewiring codex node starter --- DistTestCore/Codex/CodexStartupConfig.cs | 7 ++++--- DistTestCore/CodexSetupConfig.cs | 20 +++----------------- DistTestCore/CodexStarter.cs | 9 ++++++++- DistTestCore/DistTestCore.csproj | 1 + KubernetesWorkflow/StartupConfig.cs | 2 +- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index fdbffcf..a978b5c 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,6 +1,4 @@ -using KubernetesWorkflow; - -namespace DistTestCore.Codex +namespace DistTestCore.Codex { public class CodexStartupConfig { @@ -8,5 +6,8 @@ namespace DistTestCore.Codex public CodexLogLevel? LogLevel { get; set; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } + + //public IOnlineCodexNode? BootstrapNode { get; private set; } + //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } } } diff --git a/DistTestCore/CodexSetupConfig.cs b/DistTestCore/CodexSetupConfig.cs index 1f1a632..c7dfef8 100644 --- a/DistTestCore/CodexSetupConfig.cs +++ b/DistTestCore/CodexSetupConfig.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using KubernetesWorkflow; namespace DistTestCore { @@ -12,32 +13,17 @@ namespace DistTestCore //ICodexSetupConfig EnableMarketplace(int initialBalance); ICodexNodeGroup BringOnline(); } - - public enum Location - { - Unspecified, - BensLaptop, - BensOldGamingMachine, - } - - public class CodexSetupConfig : ICodexSetupConfig + + public class CodexSetupConfig : CodexStartupConfig, ICodexSetupConfig { private readonly CodexStarter starter; public int NumberOfNodes { get; } - public Location Location { get; private set; } - public CodexLogLevel? LogLevel { get; private set; } - //public IOnlineCodexNode? BootstrapNode { get; private set; } - public ByteSize? StorageQuota { get; private set; } - public bool MetricsEnabled { get; private set; } - //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } public CodexSetupConfig(CodexStarter starter, int numberOfNodes) { this.starter = starter; NumberOfNodes = numberOfNodes; - Location = Location.Unspecified; - MetricsEnabled = false; } public ICodexNodeGroup BringOnline() diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index f295d0a..0222925 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Codex; +using KubernetesWorkflow; using Logging; namespace DistTestCore @@ -14,7 +15,13 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetupConfig codexSetupConfig) { + var workflow = workflowCreator.CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.Add(codexSetupConfig); + var runningContainers = workflow.Start(codexSetupConfig.NumberOfNodes, codexSetupConfig.Location, new CodexContainerRecipe(), startupConfig); + + // create access objects. Easy, right? } public void DeleteAllResources() diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index ae8e964..c57463d 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -16,5 +16,6 @@ + diff --git a/KubernetesWorkflow/StartupConfig.cs b/KubernetesWorkflow/StartupConfig.cs index 7c13347..406cea9 100644 --- a/KubernetesWorkflow/StartupConfig.cs +++ b/KubernetesWorkflow/StartupConfig.cs @@ -11,7 +11,7 @@ public T Get() { - var match = configs.Single(c => c.GetType() == typeof(T)); + var match = configs.Single(c => typeof(T).IsAssignableFrom(c.GetType())); return (T)match; } } From f5c60f0bcadc5a82d3476a6ab955b7b2e04e4417 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 09:33:10 +0200 Subject: [PATCH 06/21] OneClient test passed --- DistTestCore/Codex/CodexAccess.cs | 25 ++- DistTestCore/Codex/CodexStartupConfig.cs | 4 +- DistTestCore/CodexNodeGroup.cs | 78 +++++++ .../{CodexSetupConfig.cs => CodexSetup.cs} | 22 +- DistTestCore/CodexStarter.cs | 13 +- DistTestCore/DistTest.cs | 54 ++--- DistTestCore/Http.cs | 11 +- DistTestCore/OnlineCodexNode.cs | 126 +++++++++++ DistTestCore/TestLifecycle.cs | 2 +- KubernetesWorkflow/K8sController.cs | 2 +- Tests/BasicTests/SimpleTests.cs | 196 +++++++++--------- Tests/Tests.csproj | 2 +- 12 files changed, 379 insertions(+), 156 deletions(-) create mode 100644 DistTestCore/CodexNodeGroup.cs rename DistTestCore/{CodexSetupConfig.cs => CodexSetup.cs} (73%) create mode 100644 DistTestCore/OnlineCodexNode.cs diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index a7c826e..ca6953f 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -4,13 +4,13 @@ namespace DistTestCore.Codex { public class CodexAccess { - private readonly RunningContainer runningContainer; - public CodexAccess(RunningContainer runningContainer) { - this.runningContainer = runningContainer; + Container = runningContainer; } + public RunningContainer Container { get; } + public CodexDebugResponse GetDebugInfo() { var response = Http().HttpGetJson("debug/info"); @@ -18,12 +18,27 @@ namespace DistTestCore.Codex return response; } + public string UploadFile(FileStream fileStream) + { + return Http().HttpPostStream("upload", fileStream); + } + + public Stream DownloadFile(string contentId) + { + return Http().HttpGetStream("download/" + contentId); + } + private Http Http() { - var ip = runningContainer.Pod.Cluster.GetIp(); - var port = runningContainer.ServicePorts[0].Number; + var ip = Container.Pod.Cluster.GetIp(); + var port = Container.ServicePorts[0].Number; return new Http(ip, port, baseUrl: "/api/codex/v1"); } + + public string ConnectToPeer(string peerId, string peerMultiAddress) + { + return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + } } public class CodexDebugResponse diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index a978b5c..f83d629 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,4 +1,6 @@ -namespace DistTestCore.Codex +using KubernetesWorkflow; + +namespace DistTestCore.Codex { public class CodexStartupConfig { diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs new file mode 100644 index 0000000..4ca3a6c --- /dev/null +++ b/DistTestCore/CodexNodeGroup.cs @@ -0,0 +1,78 @@ +using DistTestCore.Codex; +using KubernetesWorkflow; +using System.Collections; + +namespace DistTestCore +{ + public interface ICodexNodeGroup : IEnumerable + { + //ICodexSetup BringOffline(); + IOnlineCodexNode this[int index] { get; } + } + + public class CodexNodeGroup : ICodexNodeGroup + { + private readonly TestLifecycle lifecycle; + + public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers) + { + this.lifecycle = lifecycle; + Setup = setup; + Containers = containers; + Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c)).ToArray(); + } + + public IOnlineCodexNode this[int index] + { + get + { + return Nodes[index]; + } + } + + //public ICodexSetup BringOffline() + //{ + // //return k8SManager.BringOffline(this); + //} + + public CodexSetup Setup { get; } + public RunningContainers Containers { get; } + public OnlineCodexNode[] Nodes { get; } + + //public GethCompanionGroup? GethCompanionGroup { get; set; } + + //public CodexNodeContainer[] GetContainers() + //{ + // return Nodes.Select(n => n.Container).ToArray(); + //} + + public IEnumerator GetEnumerator() + { + return Nodes.Cast().GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return Nodes.GetEnumerator(); + } + + //public CodexNodeLog DownloadLog(IOnlineCodexNode node) + //{ + // var logDownloader = new PodLogDownloader(log, k8SManager); + // var n = (OnlineCodexNode)node; + // return logDownloader.DownloadLog(n); + //} + + public string Describe() + { + var orderNumber = Containers.RunningPod.Ip; + return $"CodexNodeGroup@{orderNumber}-{Setup.Describe()}"; + } + + private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c) + { + var access = new CodexAccess(c); + return new OnlineCodexNode(lifecycle, access, this); + } + } +} diff --git a/DistTestCore/CodexSetupConfig.cs b/DistTestCore/CodexSetup.cs similarity index 73% rename from DistTestCore/CodexSetupConfig.cs rename to DistTestCore/CodexSetup.cs index c7dfef8..ddc6ef0 100644 --- a/DistTestCore/CodexSetupConfig.cs +++ b/DistTestCore/CodexSetup.cs @@ -3,24 +3,24 @@ using KubernetesWorkflow; namespace DistTestCore { - public interface ICodexSetupConfig + public interface ICodexSetup { - ICodexSetupConfig At(Location location); - ICodexSetupConfig WithLogLevel(CodexLogLevel level); + ICodexSetup At(Location location); + ICodexSetup WithLogLevel(CodexLogLevel level); //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); - ICodexSetupConfig WithStorageQuota(ByteSize storageQuota); - ICodexSetupConfig EnableMetrics(); + ICodexSetup WithStorageQuota(ByteSize storageQuota); + ICodexSetup EnableMetrics(); //ICodexSetupConfig EnableMarketplace(int initialBalance); ICodexNodeGroup BringOnline(); } - public class CodexSetupConfig : CodexStartupConfig, ICodexSetupConfig + public class CodexSetup : CodexStartupConfig, ICodexSetup { private readonly CodexStarter starter; public int NumberOfNodes { get; } - public CodexSetupConfig(CodexStarter starter, int numberOfNodes) + public CodexSetup(CodexStarter starter, int numberOfNodes) { this.starter = starter; NumberOfNodes = numberOfNodes; @@ -31,7 +31,7 @@ namespace DistTestCore return starter.BringOnline(this); } - public ICodexSetupConfig At(Location location) + public ICodexSetup At(Location location) { Location = location; return this; @@ -43,19 +43,19 @@ namespace DistTestCore // return this; //} - public ICodexSetupConfig WithLogLevel(CodexLogLevel level) + public ICodexSetup WithLogLevel(CodexLogLevel level) { LogLevel = level; return this; } - public ICodexSetupConfig WithStorageQuota(ByteSize storageQuota) + public ICodexSetup WithStorageQuota(ByteSize storageQuota) { StorageQuota = storageQuota; return this; } - public ICodexSetupConfig EnableMetrics() + public ICodexSetup EnableMetrics() { MetricsEnabled = true; return this; diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 0222925..86ed8c3 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,27 +1,28 @@ using DistTestCore.Codex; using KubernetesWorkflow; -using Logging; namespace DistTestCore { public class CodexStarter { private readonly WorkflowCreator workflowCreator; + private readonly TestLifecycle lifecycle; - public CodexStarter(TestLog log, Configuration configuration) + public CodexStarter(TestLifecycle lifecycle, Configuration configuration) { workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + this.lifecycle = lifecycle; } - public ICodexNodeGroup BringOnline(CodexSetupConfig codexSetupConfig) + public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { var workflow = workflowCreator.CreateWorkflow(); var startupConfig = new StartupConfig(); - startupConfig.Add(codexSetupConfig); + startupConfig.Add(codexSetup); - var runningContainers = workflow.Start(codexSetupConfig.NumberOfNodes, codexSetupConfig.Location, new CodexContainerRecipe(), startupConfig); + var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); - // create access objects. Easy, right? + return new CodexNodeGroup(lifecycle, codexSetup, runningContainers); } public void DeleteAllResources() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 995fffa..9a43a7b 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -61,27 +61,27 @@ namespace DistTestCore return lifecycle.FileManager.GenerateTestFile(size); } - public ICodexSetupConfig SetupCodexNodes(int numberOfNodes) + public ICodexSetup SetupCodexNodes(int numberOfNodes) { - return new CodexSetupConfig(lifecycle.CodexStarter, numberOfNodes); + return new CodexSetup(lifecycle.CodexStarter, numberOfNodes); } private void IncludeLogsAndMetricsOnTestFailure() { - var result = TestContext.CurrentContext.Result; - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - if (IsDownloadingLogsAndMetricsEnabled()) - { - log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - k8sManager.ForEachOnlineGroup(DownloadLogs); - k8sManager.DownloadAllMetrics(); - } - else - { - log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - } - } + //var result = TestContext.CurrentContext.Result; + //if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + //{ + // if (IsDownloadingLogsAndMetricsEnabled()) + // { + // log.Log("Downloading all CodexNode logs and metrics because of test failure..."); + // k8sManager.ForEachOnlineGroup(DownloadLogs); + // k8sManager.DownloadAllMetrics(); + // } + // else + // { + // log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + // } + //} } private void Log(string msg) @@ -101,19 +101,19 @@ namespace DistTestCore private void DownloadLogs(CodexNodeGroup group) { - foreach (var node in group) - { - var downloader = new PodLogDownloader(log, k8sManager); - var n = (OnlineCodexNode)node; - downloader.DownloadLog(n); - } + //foreach (var node in group) + //{ + // var downloader = new PodLogDownloader(log, k8sManager); + // var n = (OnlineCodexNode)node; + // downloader.DownloadLog(n); + //} } - private bool IsDownloadingLogsAndMetricsEnabled() - { - var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); - } + //private bool IsDownloadingLogsAndMetricsEnabled() + //{ + // var testProperties = TestContext.CurrentContext.Test.Properties; + // return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); + //} } public static class GlobalTestFailure diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 365920b..f75768b 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,6 +1,7 @@ using Newtonsoft.Json; using NUnit.Framework; using System.Net.Http.Headers; +using Utils; namespace DistTestCore { @@ -26,8 +27,8 @@ namespace DistTestCore { using var client = GetClient(); var url = GetUrl() + route; - var result = Utils.Wait(client.GetAsync(url)); - return Utils.Wait(result.Content.ReadAsStringAsync()); + var result = Time.Wait(client.GetAsync(url)); + return Time.Wait(result.Content.ReadAsStringAsync()); }); } @@ -45,9 +46,9 @@ namespace DistTestCore var content = new StreamContent(stream); content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - var response = Utils.Wait(client.PostAsync(url, content)); + var response = Time.Wait(client.PostAsync(url, content)); - return Utils.Wait(response.Content.ReadAsStringAsync()); + return Time.Wait(response.Content.ReadAsStringAsync()); }); } @@ -58,7 +59,7 @@ namespace DistTestCore var client = GetClient(); var url = GetUrl() + route; - return Utils.Wait(client.GetStreamAsync(url)); + return Time.Wait(client.GetStreamAsync(url)); }); } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs new file mode 100644 index 0000000..a4082a8 --- /dev/null +++ b/DistTestCore/OnlineCodexNode.cs @@ -0,0 +1,126 @@ +using DistTestCore.Codex; +using NUnit.Framework; + +namespace DistTestCore +{ + public interface IOnlineCodexNode + { + CodexDebugResponse GetDebugInfo(); + ContentId UploadFile(TestFile file); + TestFile? DownloadContent(ContentId contentId); + void ConnectToPeer(IOnlineCodexNode node); + //ICodexNodeLog DownloadLog(); + //IMetricsAccess Metrics { get; } + //IMarketplaceAccess Marketplace { get; } + } + + public class OnlineCodexNode : IOnlineCodexNode + { + private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; + private const string UploadFailedMessage = "Unable to store block"; + private readonly TestLifecycle lifecycle; + + public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group) + { + this.lifecycle = lifecycle; + CodexAccess = codexAccess; + Group = group; + } + + public CodexAccess CodexAccess { get; } + public CodexNodeGroup Group { get; } + + public string GetName() + { + return $"<{CodexAccess.Container.Recipe.Name}>"; + } + + public CodexDebugResponse GetDebugInfo() + { + var response = CodexAccess.GetDebugInfo(); + Log($"Got DebugInfo with id: '{response.id}'."); + return response; + } + + public ContentId UploadFile(TestFile file) + { + Log($"Uploading file of size {file.GetFileSize()}..."); + using var fileStream = File.OpenRead(file.Filename); + var response = CodexAccess.UploadFile(fileStream); + if (response.StartsWith(UploadFailedMessage)) + { + Assert.Fail("Node failed to store block."); + } + Log($"Uploaded file. Received contentId: '{response}'."); + return new ContentId(response); + } + + public TestFile? DownloadContent(ContentId contentId) + { + Log($"Downloading for contentId: '{contentId.Id}'..."); + var file = lifecycle.FileManager.CreateEmptyTestFile(); + DownloadToFile(contentId.Id, file); + Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'."); + return file; + } + + public void ConnectToPeer(IOnlineCodexNode node) + { + var peer = (OnlineCodexNode)node; + + Log($"Connecting to peer {peer.GetName()}..."); + var peerInfo = node.GetDebugInfo(); + var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo)); + + Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); + Log($"Successfully connected to peer {peer.GetName()}."); + } + + //public ICodexNodeLog DownloadLog() + //{ + // return Group.DownloadLog(this); + //} + + public string Describe() + { + return $"{Group.Describe()} contains {GetName()}"; + } + + private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) + { + var multiAddress = peerInfo.addrs.First(); + // Todo: Is there a case where First address in list is not the way? + + if (Group == peer.Group) + { + return multiAddress; + } + + // The peer we want to connect is in a different pod. + // We must replace the default IP with the pod IP in the multiAddress. + return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip); + } + + private void DownloadToFile(string contentId, TestFile file) + { + using var fileStream = File.OpenWrite(file.Filename); + using var downloadStream = CodexAccess.DownloadFile(contentId); + downloadStream.CopyTo(fileStream); + } + + private void Log(string msg) + { + lifecycle.Log.Log($"{GetName()}: {msg}"); + } + } + + public class ContentId + { + public ContentId(string id) + { + Id = id; + } + + public string Id { get; } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 6ff92f9..7f35056 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -8,7 +8,7 @@ namespace DistTestCore { Log = new TestLog(configuration.GetLogConfig()); FileManager = new FileManager(Log, configuration); - CodexStarter = new CodexStarter(Log, configuration); + CodexStarter = new CodexStarter(this, configuration); } public TestLog Log { get; } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index df68f18..3df4549 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -187,7 +187,7 @@ namespace KubernetesWorkflow private string GetNameForPort(ContainerRecipe recipe, Port port) { - return $"P{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}"; + return $"p{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}"; } #endregion diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index e15ce98..f4214e6 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -1,4 +1,4 @@ -using CodexDistTestCore; +using DistTestCore; using NUnit.Framework; namespace Tests.BasicTests @@ -6,68 +6,6 @@ namespace Tests.BasicTests [TestFixture] public class SimpleTests : DistTest { - [Test] - public void TwoMetricsExample() - { - var group = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); - - var group2 = SetupCodexNodes(2) - .EnableMetrics() - .BringOnline(); - - var primary = group[0]; - var secondary = group[1]; - var primary2 = group2[0]; - var secondary2 = group2[1]; - - primary.ConnectToPeer(secondary); - primary2.ConnectToPeer(secondary2); - - Thread.Sleep(TimeSpan.FromMinutes(5)); - - primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - } - - [Test] - public void MarketplaceExample() - { - var group = SetupCodexNodes(4) - .WithStorageQuota(10.GB()) - .EnableMarketplace(initialBalance: 20) - .BringOnline(); - - foreach (var node in group) - { - Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); - } - - // WIP: Balance is now only ETH. - // todo: All nodes should have plenty of ETH to pay for transactions. - // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. - - - //var secondary = SetupCodexNodes(1) - // .EnableMarketplace(initialBalance: 1000) - // .BringOnline()[0]; - - //primary.ConnectToPeer(secondary); - //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); - - //var testFile = GenerateTestFile(10.MB()); - //var contentId = secondary.UploadFile(testFile); - //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, - // requiredCollateral: 10, minRequiredNumberOfNodes: 1); - - //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); - //var primaryBalance = primary.Marketplace.GetBalance(); - - //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); - //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); - } - [Test] public void OneClientTest() { @@ -82,53 +20,115 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); } - [Test] - public void TwoClientsOnePodTest() - { - var group = SetupCodexNodes(2).BringOnline(); + //[Test] + //public void TwoClientsOnePodTest() + //{ + // var group = SetupCodexNodes(2).BringOnline(); - var primary = group[0]; - var secondary = group[1]; + // var primary = group[0]; + // var secondary = group[1]; - PerformTwoClientTest(primary, secondary); - } + // PerformTwoClientTest(primary, secondary); + //} - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; + //[Test] + //public void TwoClientsTwoPodsTest() + //{ + // var primary = SetupCodexNodes(1).BringOnline()[0]; - var secondary = SetupCodexNodes(1).BringOnline()[0]; + // var secondary = SetupCodexNodes(1).BringOnline()[0]; - PerformTwoClientTest(primary, secondary); - } + // PerformTwoClientTest(primary, secondary); + //} - [Test] - [Ignore("Requires Location map to be configured for k8s cluster.")] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; + //[Test] + //[Ignore("Requires Location map to be configured for k8s cluster.")] + //public void TwoClientsTwoLocationsTest() + //{ + // var primary = SetupCodexNodes(1) + // .At(Location.BensLaptop) + // .BringOnline()[0]; - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; + // var secondary = SetupCodexNodes(1) + // .At(Location.BensOldGamingMachine) + // .BringOnline()[0]; - PerformTwoClientTest(primary, secondary); - } + // PerformTwoClientTest(primary, secondary); + //} - private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) - { - primary.ConnectToPeer(secondary); + //[Test] + //public void TwoMetricsExample() + //{ + // var group = SetupCodexNodes(2) + // .EnableMetrics() + // .BringOnline(); - var testFile = GenerateTestFile(1.MB()); + // var group2 = SetupCodexNodes(2) + // .EnableMetrics() + // .BringOnline(); - var contentId = primary.UploadFile(testFile); + // var primary = group[0]; + // var secondary = group[1]; + // var primary2 = group2[0]; + // var secondary2 = group2[1]; - var downloadedFile = secondary.DownloadContent(contentId); + // primary.ConnectToPeer(secondary); + // primary2.ConnectToPeer(secondary2); - testFile.AssertIsEqual(downloadedFile); - } + // Thread.Sleep(TimeSpan.FromMinutes(5)); + + // primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + // primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + //} + + //[Test] + //public void MarketplaceExample() + //{ + // var group = SetupCodexNodes(4) + // .WithStorageQuota(10.GB()) + // .EnableMarketplace(initialBalance: 20) + // .BringOnline(); + + // foreach (var node in group) + // { + // Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); + // } + + // // WIP: Balance is now only ETH. + // // todo: All nodes should have plenty of ETH to pay for transactions. + // // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. + + + // //var secondary = SetupCodexNodes(1) + // // .EnableMarketplace(initialBalance: 1000) + // // .BringOnline()[0]; + + // //primary.ConnectToPeer(secondary); + // //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); + + // //var testFile = GenerateTestFile(10.MB()); + // //var contentId = secondary.UploadFile(testFile); + // //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, + // // requiredCollateral: 10, minRequiredNumberOfNodes: 1); + + // //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); + // //var primaryBalance = primary.Marketplace.GetBalance(); + + // //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); + // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + //} + + //private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + //{ + // primary.ConnectToPeer(secondary); + + // var testFile = GenerateTestFile(1.MB()); + + // var contentId = primary.UploadFile(testFile); + + // var downloadedFile = secondary.DownloadContent(contentId); + + // testFile.AssertIsEqual(downloadedFile); + //} } } diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index fc5152f..136951d 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -13,7 +13,7 @@ - + From 56063bbbf1dfc109b7c0df18e9335fa9b698c28c Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 10:11:33 +0200 Subject: [PATCH 07/21] two-client tests pass --- DistTestCore/Codex/CodexAccess.cs | 2 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 9 ++- KubernetesWorkflow/K8sCluster.cs | 43 ++++++------ KubernetesWorkflow/K8sController.cs | 2 +- Tests/BasicTests/SimpleTests.cs | 71 ++++++++++---------- 5 files changed, 67 insertions(+), 60 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index ca6953f..d0c0f72 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -30,7 +30,7 @@ namespace DistTestCore.Codex private Http Http() { - var ip = Container.Pod.Cluster.GetIp(); + var ip = Container.Pod.Cluster.IP; var port = Container.ServicePorts[0].Number; return new Http(ip, port, baseUrl: "/api/codex/v1"); } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index fc06f33..60ea91e 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -14,7 +14,14 @@ Initialize(config); - return new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + + exposedPorts.Clear(); + internalPorts.Clear(); + envVars.Clear(); + this.factory = null!; + + return recipe; } protected abstract string Image { get; } diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index c823a09..4d5a772 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -4,40 +4,21 @@ namespace KubernetesWorkflow { public class K8sCluster { - private KubernetesClientConfiguration? config; - public K8sCluster(Configuration configuration) { Configuration = configuration; } public Configuration Configuration { get; } + public string IP { get; private set; } = string.Empty; public KubernetesClientConfiguration GetK8sClientConfig() { - if (config != null) return config; - - if (Configuration.KubeConfigFile != null) - { - config = KubernetesClientConfiguration.BuildConfigFromConfigFile(Configuration.KubeConfigFile); - } - else - { - config = KubernetesClientConfiguration.BuildDefaultConfig(); - } - + var config = GetConfig(); + UpdateIp(config); return config; } - public string GetIp() - { - var c = GetK8sClientConfig(); - - var host = c.Host.Replace("https://", ""); - - return host.Substring(0, host.IndexOf(':')); - } - public string GetNodeLabelForLocation(Location location) { if (location == Location.Unspecified) return string.Empty; @@ -53,5 +34,23 @@ namespace KubernetesWorkflow { return Configuration.RetryDelay; } + + private KubernetesClientConfiguration GetConfig() + { + if (Configuration.KubeConfigFile != null) + { + return KubernetesClientConfiguration.BuildConfigFromConfigFile(Configuration.KubeConfigFile); + } + else + { + return KubernetesClientConfiguration.BuildDefaultConfig(); + } + } + + private void UpdateIp(KubernetesClientConfiguration config) + { + var host = config.Host.Replace("https://", ""); + IP = host.Substring(0, host.IndexOf(':')); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 3df4549..295397b 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -227,7 +227,7 @@ namespace KubernetesWorkflow { return new V1ObjectMeta { - Name = "deploy-" + workflowNumberSource.WorkflowNumber, + Name = "service-" + workflowNumberSource.WorkflowNumber, NamespaceProperty = K8sNamespace }; } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index f4214e6..4c33425 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using KubernetesWorkflow; using NUnit.Framework; namespace Tests.BasicTests @@ -20,41 +21,41 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); } - //[Test] - //public void TwoClientsOnePodTest() - //{ - // var group = SetupCodexNodes(2).BringOnline(); + [Test] + public void TwoClientsOnePodTest() + { + var group = SetupCodexNodes(2).BringOnline(); - // var primary = group[0]; - // var secondary = group[1]; + var primary = group[0]; + var secondary = group[1]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } - //[Test] - //public void TwoClientsTwoPodsTest() - //{ - // var primary = SetupCodexNodes(1).BringOnline()[0]; + [Test] + public void TwoClientsTwoPodsTest() + { + var primary = SetupCodexNodes(1).BringOnline()[0]; - // var secondary = SetupCodexNodes(1).BringOnline()[0]; + var secondary = SetupCodexNodes(1).BringOnline()[0]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } - //[Test] - //[Ignore("Requires Location map to be configured for k8s cluster.")] - //public void TwoClientsTwoLocationsTest() - //{ - // var primary = SetupCodexNodes(1) - // .At(Location.BensLaptop) - // .BringOnline()[0]; + [Test] + [Ignore("Requires Location map to be configured for k8s cluster.")] + public void TwoClientsTwoLocationsTest() + { + var primary = SetupCodexNodes(1) + .At(Location.BensLaptop) + .BringOnline()[0]; - // var secondary = SetupCodexNodes(1) - // .At(Location.BensOldGamingMachine) - // .BringOnline()[0]; + var secondary = SetupCodexNodes(1) + .At(Location.BensOldGamingMachine) + .BringOnline()[0]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } //[Test] //public void TwoMetricsExample() @@ -118,17 +119,17 @@ namespace Tests.BasicTests // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); //} - //private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) - //{ - // primary.ConnectToPeer(secondary); + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + primary.ConnectToPeer(secondary); - // var testFile = GenerateTestFile(1.MB()); + var testFile = GenerateTestFile(1.MB()); - // var contentId = primary.UploadFile(testFile); + var contentId = primary.UploadFile(testFile); - // var downloadedFile = secondary.DownloadContent(contentId); + var downloadedFile = secondary.DownloadContent(contentId); - // testFile.AssertIsEqual(downloadedFile); - //} + testFile.AssertIsEqual(downloadedFile); + } } } From cbf0fbf5b572d684a17a6bc59c888c15f97bf469 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 11:07:36 +0200 Subject: [PATCH 08/21] Implements restart test. --- DistTestCore/CodexNodeGroup.cs | 25 ++++++--- DistTestCore/CodexStarter.cs | 15 +++++- KubernetesWorkflow/K8sController.cs | 63 +++++++++++++++++----- KubernetesWorkflow/RunningPod.cs | 6 ++- KubernetesWorkflow/StartupWorkflow.cs | 8 +++ KubernetesWorkflow/WorkflowCreator.cs | 5 +- KubernetesWorkflow/WorkflowNumberSource.cs | 5 +- Tests/BasicTests/SimpleTests.cs | 25 +++++++-- 8 files changed, 121 insertions(+), 31 deletions(-) diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 4ca3a6c..e486162 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -6,7 +6,7 @@ namespace DistTestCore { public interface ICodexNodeGroup : IEnumerable { - //ICodexSetup BringOffline(); + ICodexSetup BringOffline(); IOnlineCodexNode this[int index] { get; } } @@ -30,14 +30,23 @@ namespace DistTestCore } } - //public ICodexSetup BringOffline() - //{ - // //return k8SManager.BringOffline(this); - //} + public ICodexSetup BringOffline() + { + var result = Setup; + var containers = Containers; - public CodexSetup Setup { get; } - public RunningContainers Containers { get; } - public OnlineCodexNode[] Nodes { get; } + // Clear everything. Prevent accidental use. + Setup = null!; + Containers = null!; + Nodes = Array.Empty(); + + lifecycle.CodexStarter.BringOffline(containers); + return result; + } + + public CodexSetup Setup { get; private set; } + public RunningContainers Containers { get; private set; } + public OnlineCodexNode[] Nodes { get; private set; } //public GethCompanionGroup? GethCompanionGroup { get; set; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 86ed8c3..8ad0aa8 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -16,7 +16,7 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - var workflow = workflowCreator.CreateWorkflow(); + var workflow = CreateWorkflow(); var startupConfig = new StartupConfig(); startupConfig.Add(codexSetup); @@ -25,10 +25,21 @@ namespace DistTestCore return new CodexNodeGroup(lifecycle, codexSetup, runningContainers); } + public void BringOffline(RunningContainers runningContainers) + { + var workflow = CreateWorkflow(); + workflow.Stop(runningContainers); + } + public void DeleteAllResources() { - var workflow = workflowCreator.CreateWorkflow(); + var workflow = CreateWorkflow(); workflow.DeleteAllResources(); } + + private StartupWorkflow CreateWorkflow() + { + return workflowCreator.CreateWorkflow(); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 295397b..a3b25fa 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -28,11 +28,19 @@ namespace KubernetesWorkflow { EnsureTestNamespace(); - CreateDeployment(containerRecipes, location); - var servicePortsMap = CreateService(containerRecipes); + var deploymentName = CreateDeployment(containerRecipes, location); + var (serviceName, servicePortsMap) = CreateService(containerRecipes); var (podName, podIp) = FetchNewPod(); - return new RunningPod(cluster, podName, podIp, servicePortsMap); + return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap); + } + + public void Stop(RunningPod pod) + { + if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName); + DeleteDeployment(pod.DeploymentName); + WaitUntilDeploymentOffline(pod.DeploymentName); + WaitUntilPodOffline(pod.Name); } public void DeleteAllResources() @@ -83,7 +91,7 @@ namespace KubernetesWorkflow #region Deployment management - private void CreateDeployment(ContainerRecipe[] containerRecipes, Location location) + private string CreateDeployment(ContainerRecipe[] containerRecipes, Location location) { var deploymentSpec = new V1Deployment { @@ -112,7 +120,15 @@ namespace KubernetesWorkflow }; client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - WaitUntilDeploymentCreated(deploymentSpec); + WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); + + return deploymentSpec.Metadata.Name; + } + + private void DeleteDeployment(string deploymentName) + { + client.DeleteNamespacedDeployment(deploymentName, K8sNamespace); + WaitUntilDeploymentOffline(deploymentName); } private IDictionary CreateNodeSelector(Location location) @@ -194,7 +210,7 @@ namespace KubernetesWorkflow #region Service management - private Dictionary CreateService(ContainerRecipe[] containerRecipes) + private (string, Dictionary) CreateService(ContainerRecipe[] containerRecipes) { var result = new Dictionary(); @@ -204,7 +220,7 @@ namespace KubernetesWorkflow { // None of these container-recipes wish to expose anything via a serice port. // So, we don't have to create a service. - return result; + return (string.Empty, result); } var serviceSpec = new V1Service @@ -220,7 +236,13 @@ namespace KubernetesWorkflow }; client.CreateNamespacedService(serviceSpec, K8sNamespace); - return result; + + return (serviceSpec.Metadata.Name, result); + } + + private void DeleteService(string serviceName) + { + client.DeleteNamespacedService(serviceName, K8sNamespace); } private V1ObjectMeta CreateServiceMetadata() @@ -279,11 +301,6 @@ namespace KubernetesWorkflow WaitUntil(() => !IsTestNamespaceOnline()); } - private void WaitUntilDeploymentCreated(V1Deployment deploymentSpec) - { - WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); - } - private void WaitUntilDeploymentOnline(string deploymentName) { WaitUntil(() => @@ -293,6 +310,26 @@ namespace KubernetesWorkflow }); } + private void WaitUntilDeploymentOffline(string deploymentName) + { + WaitUntil(() => + { + var deployments = client.ListNamespacedDeployment(K8sNamespace); + var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); + return deployment == null || deployment.Status.AvailableReplicas == 0; + }); + } + + private void WaitUntilPodOffline(string podName) + { + WaitUntil(() => + { + var pods = client.ListNamespacedPod(K8sNamespace).Items; + var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName); + return pod == null; + }); + } + private void WaitUntil(Func predicate) { var start = DateTime.UtcNow; diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index 63378e2..b676903 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -4,17 +4,21 @@ { private readonly Dictionary servicePortMap; - public RunningPod(K8sCluster cluster, string name, string ip, Dictionary servicePortMap) + public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary servicePortMap) { Cluster = cluster; Name = name; Ip = ip; + DeploymentName = deploymentName; + ServiceName = serviceName; this.servicePortMap = servicePortMap; } public K8sCluster Cluster { get; } public string Name { get; } public string Ip { get; } + internal string DeploymentName { get; } + internal string ServiceName { get; } public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) { diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 8486590..c3faa50 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -26,6 +26,14 @@ }); } + public void Stop(RunningContainers runningContainers) + { + K8s(controller => + { + controller.Stop(runningContainers.RunningPod); + }); + } + public void DeleteAllResources() { K8s(controller => diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 88af0e8..51c1c29 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -6,6 +6,7 @@ namespace KubernetesWorkflow { private readonly NumberSource numberSource = new NumberSource(0); private readonly NumberSource servicePortNumberSource = new NumberSource(30001); + private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; @@ -16,7 +17,9 @@ namespace KubernetesWorkflow public StartupWorkflow CreateWorkflow() { - var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), servicePortNumberSource); + var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), + servicePortNumberSource, + containerNumberSource); return new StartupWorkflow(workflowNumberSource, cluster, knownPods); } diff --git a/KubernetesWorkflow/WorkflowNumberSource.cs b/KubernetesWorkflow/WorkflowNumberSource.cs index 018b97b..8cbab34 100644 --- a/KubernetesWorkflow/WorkflowNumberSource.cs +++ b/KubernetesWorkflow/WorkflowNumberSource.cs @@ -4,13 +4,14 @@ namespace KubernetesWorkflow { public class WorkflowNumberSource { - private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly NumberSource servicePortNumberSource; + private readonly NumberSource containerNumberSource; - public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource) + public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource) { WorkflowNumber = workflowNumber; this.servicePortNumberSource = servicePortNumberSource; + this.containerNumberSource = containerNumberSource; } public int WorkflowNumber { get; } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 4c33425..a723471 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -12,13 +12,19 @@ namespace Tests.BasicTests { var primary = SetupCodexNodes(1).BringOnline()[0]; - var testFile = GenerateTestFile(1.MB()); + PerformOneClientTest(primary); + } - var contentId = primary.UploadFile(testFile); + [Test] + public void RestartTest() + { + var group = SetupCodexNodes(1).BringOnline(); - var downloadedFile = primary.DownloadContent(contentId); + var setup = group.BringOffline(); - testFile.AssertIsEqual(downloadedFile); + var primary = setup.BringOnline()[0]; + + PerformOneClientTest(primary); } [Test] @@ -119,6 +125,17 @@ namespace Tests.BasicTests // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); //} + private void PerformOneClientTest(IOnlineCodexNode primary) + { + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = primary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { primary.ConnectToPeer(secondary); From 7eab4840efc2e4a7ac9e3cc194c46e1bb8b9a52b Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 11:30:19 +0200 Subject: [PATCH 09/21] Log accessing test passes --- DistTestCore/CodexLogs/CodexNodeLog.cs | 35 ++++++++++++++++++++ DistTestCore/CodexLogs/LogDownloadHandler.cs | 35 ++++++++++++++++++++ DistTestCore/CodexStarter.cs | 8 +++++ DistTestCore/OnlineCodexNode.cs | 11 +++--- DistTestCore/TestLifecycle.cs | 14 +++++++- KubernetesWorkflow/K8sController.cs | 6 ++++ KubernetesWorkflow/StartupWorkflow.cs | 12 +++++++ Tests/BasicTests/SimpleTests.cs | 15 +++++++++ 8 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 DistTestCore/CodexLogs/CodexNodeLog.cs create mode 100644 DistTestCore/CodexLogs/LogDownloadHandler.cs diff --git a/DistTestCore/CodexLogs/CodexNodeLog.cs b/DistTestCore/CodexLogs/CodexNodeLog.cs new file mode 100644 index 0000000..5003bf8 --- /dev/null +++ b/DistTestCore/CodexLogs/CodexNodeLog.cs @@ -0,0 +1,35 @@ +using Logging; +using NUnit.Framework; + +namespace DistTestCore.CodexLogs +{ + public interface ICodexNodeLog + { + void AssertLogContains(string expectedString); + } + + public class CodexNodeLog : ICodexNodeLog + { + private readonly LogFile logFile; + + public CodexNodeLog(LogFile logFile) + { + this.logFile = logFile; + } + + public void AssertLogContains(string expectedString) + { + using var file = File.OpenRead(logFile.FullFilename); + using var streamReader = new StreamReader(file); + + var line = streamReader.ReadLine(); + while (line != null) + { + if (line.Contains(expectedString)) return; + line = streamReader.ReadLine(); + } + + Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}"); + } + } +} diff --git a/DistTestCore/CodexLogs/LogDownloadHandler.cs b/DistTestCore/CodexLogs/LogDownloadHandler.cs new file mode 100644 index 0000000..3848b28 --- /dev/null +++ b/DistTestCore/CodexLogs/LogDownloadHandler.cs @@ -0,0 +1,35 @@ +using KubernetesWorkflow; +using Logging; + +namespace DistTestCore.CodexLogs +{ + public class LogDownloadHandler : ILogHandler + { + private readonly string description; + private readonly LogFile log; + + public LogDownloadHandler(string description, LogFile log) + { + this.description = description; + this.log = log; + } + + public CodexNodeLog CreateCodexNodeLog() + { + return new CodexNodeLog(log); + } + + public void Log(Stream stream) + { + log.Write($"{description} -->> {log.FilenameWithoutPath}"); + log.WriteRaw(description); + var reader = new StreamReader(stream); + var line = reader.ReadLine(); + while (line != null) + { + log.WriteRaw(line); + line = reader.ReadLine(); + } + } + } +} diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 8ad0aa8..41c685f 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,5 +1,7 @@ using DistTestCore.Codex; +using DistTestCore.CodexLogs; using KubernetesWorkflow; +using Nethereum.Merkle.Patricia; namespace DistTestCore { @@ -37,6 +39,12 @@ namespace DistTestCore workflow.DeleteAllResources(); } + public void DownloadLog(RunningContainer container, ILogHandler logHandler) + { + var workflow = CreateWorkflow(); + workflow.DownloadContainerLog(container, logHandler); + } + private StartupWorkflow CreateWorkflow() { return workflowCreator.CreateWorkflow(); diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index a4082a8..9ecaef5 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.CodexLogs; using NUnit.Framework; namespace DistTestCore @@ -9,7 +10,7 @@ namespace DistTestCore ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); - //ICodexNodeLog DownloadLog(); + ICodexNodeLog DownloadLog(); //IMetricsAccess Metrics { get; } //IMarketplaceAccess Marketplace { get; } } @@ -76,10 +77,10 @@ namespace DistTestCore Log($"Successfully connected to peer {peer.GetName()}."); } - //public ICodexNodeLog DownloadLog() - //{ - // return Group.DownloadLog(this); - //} + public ICodexNodeLog DownloadLog() + { + return lifecycle.DownloadLog(this); + } public string Describe() { diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 7f35056..95f4508 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,5 @@ -using Logging; +using DistTestCore.CodexLogs; +using Logging; namespace DistTestCore { @@ -20,5 +21,16 @@ namespace DistTestCore CodexStarter.DeleteAllResources(); FileManager.DeleteAllTestFiles(); } + + public ICodexNodeLog DownloadLog(OnlineCodexNode node) + { + var subFile = Log.CreateSubfile(); + var description = node.Describe(); + var handler = new LogDownloadHandler(description, subFile); + + CodexStarter.DownloadLog(node.CodexAccess.Container, handler); + + return new CodexNodeLog(subFile); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index a3b25fa..eb492a9 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -43,6 +43,12 @@ namespace KubernetesWorkflow WaitUntilPodOffline(pod.Name); } + public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) + { + var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); + logHandler.Log(stream); + } + public void DeleteAllResources() { DeleteNamespace(); diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index c3faa50..9dff328 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -34,6 +34,14 @@ }); } + public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler) + { + K8s(controller => + { + controller.DownloadPodLog(container.Pod, container.Recipe, logHandler); + }); + } + public void DeleteAllResources() { K8s(controller => @@ -72,6 +80,10 @@ controller.Dispose(); return result; } + } + public interface ILogHandler + { + void Log(Stream log); } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index a723471..6f6ce14 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using DistTestCore.Codex; using KubernetesWorkflow; using NUnit.Framework; @@ -63,6 +64,20 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } + [Test] + public void CodexLogExample() + { + var primary = SetupCodexNodes(1) + .WithLogLevel(CodexLogLevel.Trace) + .BringOnline()[0]; + + primary.UploadFile(GenerateTestFile(5.MB())); + + var log = primary.DownloadLog(); + + log.AssertLogContains("Uploaded file"); + } + //[Test] //public void TwoMetricsExample() //{ From 31e034ab6776f285ed70cab65e49193602830e0c Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 11:53:54 +0200 Subject: [PATCH 10/21] Restores automatic log download on test failure --- CodexDistTestCore/PodLogDownloader.cs | 9 ---- .../CodexNodeLog.cs | 2 +- ...ownloadLogsAndMetricsOnFailureAttribute.cs | 15 ++++++ .../LogDownloadHandler.cs | 2 +- DistTestCore/CodexNodeGroup.cs | 7 ++- DistTestCore/CodexStarter.cs | 26 +++++++-- DistTestCore/DistTest.cs | 54 +++++++++---------- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/TestLifecycle.cs | 3 +- 9 files changed, 71 insertions(+), 49 deletions(-) rename DistTestCore/{CodexLogs => CodexLogsAndMetrics}/CodexNodeLog.cs (95%) create mode 100644 DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs rename DistTestCore/{CodexLogs => CodexLogsAndMetrics}/LogDownloadHandler.cs (95%) diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs index 4df84d2..e09a57c 100644 --- a/CodexDistTestCore/PodLogDownloader.cs +++ b/CodexDistTestCore/PodLogDownloader.cs @@ -7,15 +7,6 @@ namespace CodexDistTestCore void Log(Stream log); } - [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute - { - public DontDownloadLogsAndMetricsOnFailureAttribute() - : base(Timing.UseLongTimeoutsKey) - { - } - } - public class PodLogDownloader { public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure"; diff --git a/DistTestCore/CodexLogs/CodexNodeLog.cs b/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs similarity index 95% rename from DistTestCore/CodexLogs/CodexNodeLog.cs rename to DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs index 5003bf8..a4a9cb0 100644 --- a/DistTestCore/CodexLogs/CodexNodeLog.cs +++ b/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs @@ -1,7 +1,7 @@ using Logging; using NUnit.Framework; -namespace DistTestCore.CodexLogs +namespace DistTestCore.CodexLogsAndMetrics { public interface ICodexNodeLog { diff --git a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs b/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs new file mode 100644 index 0000000..0cf1cbe --- /dev/null +++ b/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs @@ -0,0 +1,15 @@ +using NUnit.Framework; + +namespace DistTestCore.CodexLogsAndMetrics +{ + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute + { + public const string DontDownloadKey = "DontDownloadLogsAndMetrics"; + + public DontDownloadLogsAndMetricsOnFailureAttribute() + : base(DontDownloadKey) + { + } + } +} diff --git a/DistTestCore/CodexLogs/LogDownloadHandler.cs b/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs similarity index 95% rename from DistTestCore/CodexLogs/LogDownloadHandler.cs rename to DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs index 3848b28..59c14d8 100644 --- a/DistTestCore/CodexLogs/LogDownloadHandler.cs +++ b/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs @@ -1,7 +1,7 @@ using KubernetesWorkflow; using Logging; -namespace DistTestCore.CodexLogs +namespace DistTestCore.CodexLogsAndMetrics { public class LogDownloadHandler : ILogHandler { diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index e486162..2c5c312 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -32,15 +32,14 @@ namespace DistTestCore public ICodexSetup BringOffline() { - var result = Setup; - var containers = Containers; + lifecycle.CodexStarter.BringOffline(this); + var result = Setup; // Clear everything. Prevent accidental use. Setup = null!; - Containers = null!; Nodes = Array.Empty(); + Containers = null!; - lifecycle.CodexStarter.BringOffline(containers); return result; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 41c685f..d7bc0ab 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,7 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogs; using KubernetesWorkflow; -using Nethereum.Merkle.Patricia; namespace DistTestCore { @@ -16,27 +14,40 @@ namespace DistTestCore this.lifecycle = lifecycle; } + public List RunningGroups { get; } = new List(); + public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { + Log($"Starting {codexSetup.Describe()}..."); + var workflow = CreateWorkflow(); var startupConfig = new StartupConfig(); startupConfig.Add(codexSetup); var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); - return new CodexNodeGroup(lifecycle, codexSetup, runningContainers); + var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers); + RunningGroups.Add(group); + + Log($"Started at '{group.Containers.RunningPod.Ip}'"); + return group; } - public void BringOffline(RunningContainers runningContainers) + public void BringOffline(CodexNodeGroup group) { + Log($"Stopping {group.Describe()}..."); var workflow = CreateWorkflow(); - workflow.Stop(runningContainers); + workflow.Stop(group.Containers); + RunningGroups.Remove(group); + Log("Stopped."); } public void DeleteAllResources() { var workflow = CreateWorkflow(); workflow.DeleteAllResources(); + + RunningGroups.Clear(); } public void DownloadLog(RunningContainer container, ILogHandler logHandler) @@ -49,5 +60,10 @@ namespace DistTestCore { return workflowCreator.CreateWorkflow(); } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } } } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 9a43a7b..d1830ba 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using DistTestCore.CodexLogsAndMetrics; +using NUnit.Framework; namespace DistTestCore { @@ -68,20 +69,20 @@ namespace DistTestCore private void IncludeLogsAndMetricsOnTestFailure() { - //var result = TestContext.CurrentContext.Result; - //if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - //{ - // if (IsDownloadingLogsAndMetricsEnabled()) - // { - // log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - // k8sManager.ForEachOnlineGroup(DownloadLogs); - // k8sManager.DownloadAllMetrics(); - // } - // else - // { - // log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - // } - //} + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + if (IsDownloadingLogsAndMetricsEnabled()) + { + Log("Downloading all CodexNode logs and metrics because of test failure..."); + DownloadAllLogs(); + //k8sManager.DownloadAllMetrics(); + } + else + { + Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + } + } } private void Log(string msg) @@ -99,21 +100,20 @@ namespace DistTestCore lifecycle = new TestLifecycle(new Configuration()); } - private void DownloadLogs(CodexNodeGroup group) + private void DownloadAllLogs() { - //foreach (var node in group) - //{ - // var downloader = new PodLogDownloader(log, k8sManager); - // var n = (OnlineCodexNode)node; - // downloader.DownloadLog(n); - //} + var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); + foreach (var node in allNodes) + { + lifecycle.DownloadLog(node); + } } - //private bool IsDownloadingLogsAndMetricsEnabled() - //{ - // var testProperties = TestContext.CurrentContext.Test.Properties; - // return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); - //} + private bool IsDownloadingLogsAndMetricsEnabled() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey); + } } public static class GlobalTestFailure diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 9ecaef5..8f85f91 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogs; +using DistTestCore.CodexLogsAndMetrics; using NUnit.Framework; namespace DistTestCore diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 95f4508..1267216 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogs; +using DistTestCore.CodexLogsAndMetrics; using Logging; namespace DistTestCore @@ -28,6 +28,7 @@ namespace DistTestCore var description = node.Describe(); var handler = new LogDownloadHandler(description, subFile); + Log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); CodexStarter.DownloadLog(node.CodexAccess.Container, handler); return new CodexNodeLog(subFile); From 33a3f85136d48f777da2e3c82cdc9b0eabf994b6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 14:36:17 +0200 Subject: [PATCH 11/21] Metrics example test passes --- DistTestCore/Codex/CodexContainerRecipe.cs | 4 +- DistTestCore/CodexNodeFactory.cs | 28 +++ DistTestCore/CodexNodeGroup.cs | 11 +- DistTestCore/CodexStarter.cs | 38 +++- DistTestCore/Metrics/MetricsAccess.cs | 65 ++++++ DistTestCore/Metrics/MetricsAccessFactory.cs | 33 +++ DistTestCore/Metrics/MetricsDownloader.cs | 98 +++++++++ DistTestCore/Metrics/MetricsQuery.cs | 195 ++++++++++++++++++ .../Metrics/PrometheusContainerRecipe.cs | 17 ++ .../Metrics/PrometheusStartupConfig.cs | 12 ++ DistTestCore/OnlineCodexNode.cs | 7 +- DistTestCore/PrometheusStarter.cs | 65 ++++++ DistTestCore/TestLifecycle.cs | 9 +- KubernetesWorkflow/ContainerRecipe.cs | 9 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 16 +- KubernetesWorkflow/K8sController.cs | 2 +- KubernetesWorkflow/RecipeComponentFactory.cs | 4 +- KubernetesWorkflow/RunningContainers.cs | 5 + Tests/BasicTests/SimpleTests.cs | 38 ++-- 19 files changed, 604 insertions(+), 52 deletions(-) create mode 100644 DistTestCore/CodexNodeFactory.cs create mode 100644 DistTestCore/Metrics/MetricsAccess.cs create mode 100644 DistTestCore/Metrics/MetricsAccessFactory.cs create mode 100644 DistTestCore/Metrics/MetricsDownloader.cs create mode 100644 DistTestCore/Metrics/MetricsQuery.cs create mode 100644 DistTestCore/Metrics/PrometheusContainerRecipe.cs create mode 100644 DistTestCore/Metrics/PrometheusStartupConfig.cs create mode 100644 DistTestCore/PrometheusStarter.cs diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 9c7b87d..e49dd84 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -4,6 +4,8 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { + public const string MetricsPortTag = "metrics_port"; + protected override string Image => "thatbenbierens/nim-codex:sha-b204837"; protected override void Initialize(StartupConfig startupConfig) @@ -28,7 +30,7 @@ namespace DistTestCore.Codex if (config.MetricsEnabled) { AddEnvVar("METRICS_ADDR", "0.0.0.0"); - AddInternalPortAndVar("METRICS_PORT"); + AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag); } } } diff --git a/DistTestCore/CodexNodeFactory.cs b/DistTestCore/CodexNodeFactory.cs new file mode 100644 index 0000000..3dce8aa --- /dev/null +++ b/DistTestCore/CodexNodeFactory.cs @@ -0,0 +1,28 @@ +using DistTestCore.Codex; +using DistTestCore.Metrics; + +namespace DistTestCore +{ + public interface ICodexNodeFactory + { + OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group); + } + + public class CodexNodeFactory : ICodexNodeFactory + { + private readonly TestLifecycle lifecycle; + private readonly IMetricsAccessFactory metricsAccessFactory; + + public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory) + { + this.lifecycle = lifecycle; + this.metricsAccessFactory = metricsAccessFactory; + } + + public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group) + { + var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container); + return new OnlineCodexNode(lifecycle, access, group, metricsAccess); + } + } +} diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 2c5c312..da4313f 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -14,12 +14,12 @@ namespace DistTestCore { private readonly TestLifecycle lifecycle; - public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers) + public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory) { this.lifecycle = lifecycle; Setup = setup; Containers = containers; - Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c)).ToArray(); + Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray(); } public IOnlineCodexNode this[int index] @@ -73,14 +73,13 @@ namespace DistTestCore public string Describe() { - var orderNumber = Containers.RunningPod.Ip; - return $"CodexNodeGroup@{orderNumber}-{Setup.Describe()}"; + return $"CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}"; } - private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c) + private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { var access = new CodexAccess(c); - return new OnlineCodexNode(lifecycle, access, this); + return factory.CreateOnlineCodexNode(access, this); } } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index d7bc0ab..65c9a32 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -5,31 +5,27 @@ namespace DistTestCore { public class CodexStarter { - private readonly WorkflowCreator workflowCreator; private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; - public CodexStarter(TestLifecycle lifecycle, Configuration configuration) + public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { - workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; } public List RunningGroups { get; } = new List(); public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - Log($"Starting {codexSetup.Describe()}..."); + var containers = StartCodexContainers(codexSetup); - var workflow = CreateWorkflow(); - var startupConfig = new StartupConfig(); - startupConfig.Add(codexSetup); + var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); - var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); + var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory); - var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers); - RunningGroups.Add(group); + var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - Log($"Started at '{group.Containers.RunningPod.Ip}'"); return group; } @@ -55,6 +51,26 @@ namespace DistTestCore var workflow = CreateWorkflow(); workflow.DownloadContainerLog(container, logHandler); } + + private RunningContainers StartCodexContainers(CodexSetup codexSetup) + { + Log($"Starting {codexSetup.Describe()}..."); + + var workflow = CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.Add(codexSetup); + + return workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); + } + + private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory) + { + var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); + RunningGroups.Add(group); + + Log($"Started at '{group.Containers.RunningPod.Ip}'"); + return group; + } private StartupWorkflow CreateWorkflow() { diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs new file mode 100644 index 0000000..3287ea7 --- /dev/null +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -0,0 +1,65 @@ +using KubernetesWorkflow; +using NUnit.Framework; +using NUnit.Framework.Constraints; +using Utils; + +namespace DistTestCore.Metrics +{ + public interface IMetricsAccess + { + void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); + } + + public class MetricsUnavailable : IMetricsAccess + { + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } + } + + public class MetricsAccess : IMetricsAccess + { + private readonly MetricsQuery query; + private readonly RunningContainer node; + + public MetricsAccess(MetricsQuery query, RunningContainer node) + { + this.query = query; + this.node = node; + } + + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + var metricSet = GetMetricWithTimeout(metricName); + var metricValue = metricSet.Values[0].Value; + Assert.That(metricValue, constraint, message); + } + + private MetricsSet GetMetricWithTimeout(string metricName) + { + var start = DateTime.UtcNow; + + while (true) + { + var mostRecent = GetMostRecent(metricName); + if (mostRecent != null) return mostRecent; + if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) + { + Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); + throw new TimeoutException(); + } + + Time.Sleep(TimeSpan.FromSeconds(2)); + } + } + + private MetricsSet? GetMostRecent(string metricName) + { + var result = query.GetMostRecent(metricName, node); + if (result == null) return null; + return result.Sets.LastOrDefault(); + } + } +} diff --git a/DistTestCore/Metrics/MetricsAccessFactory.cs b/DistTestCore/Metrics/MetricsAccessFactory.cs new file mode 100644 index 0000000..fcf5dfb --- /dev/null +++ b/DistTestCore/Metrics/MetricsAccessFactory.cs @@ -0,0 +1,33 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Metrics +{ + public interface IMetricsAccessFactory + { + IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer); + } + + public class MetricsUnavailableAccessFactory : IMetricsAccessFactory + { + public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) + { + return new MetricsUnavailable(); + } + } + + public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory + { + private readonly RunningContainers prometheusContainer; + + public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer) + { + this.prometheusContainer = prometheusContainer; + } + + public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) + { + var query = new MetricsQuery(prometheusContainer); + return new MetricsAccess(query, codexContainer); + } + } +} diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs new file mode 100644 index 0000000..3d79752 --- /dev/null +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -0,0 +1,98 @@ +using Logging; +using System.Globalization; + +namespace DistTestCore.Metrics +{ + public class MetricsDownloader + { + private readonly TestLog log; + private readonly Dictionary activePrometheuses; + + public MetricsDownloader(TestLog log, Dictionary activePrometheuses) + { + this.log = log; + this.activePrometheuses = activePrometheuses; + } + + public void DownloadAllMetrics() + { + foreach (var pair in activePrometheuses) + { + DownloadAllMetrics(pair.Key, pair.Value); + } + } + + private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) + { + foreach (var node in nodes) + { + DownloadAllMetricsForNode(query, node); + } + } + + private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) + { + var metrics = query.GetAllMetricsForNode(node.CodexAccess.Container); + if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; + + var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); + var map = CreateValueMap(metrics); + + WriteToFile(node.GetName(), headers, map); + } + + private void WriteToFile(string nodeName, string[] headers, Dictionary> map) + { + var file = log.CreateSubfile("csv"); + log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); + + file.WriteRaw(string.Join(",", headers)); + + foreach (var pair in map) + { + file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); + } + } + + private Dictionary> CreateValueMap(Metrics metrics) + { + var map = CreateForAllTimestamps(metrics); + foreach (var metric in metrics.Sets) + { + AddToMap(map, metric); + } + return map; + + } + + private Dictionary> CreateForAllTimestamps(Metrics metrics) + { + var result = new Dictionary>(); + var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); + foreach (var timestamp in timestamps) result.Add(timestamp, new List()); + return result; + } + + private void AddToMap(Dictionary> map, MetricsSet metric) + { + foreach (var key in map.Keys) + { + map[key].Add(GetValueAtTimestamp(key, metric)); + } + } + + private string GetValueAtTimestamp(DateTime key, MetricsSet metric) + { + var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); + if (value == null) return ""; + return value.Value.ToString(CultureInfo.InvariantCulture); + } + + private string FormatTimestamp(DateTime key) + { + var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + var diff = key - origin; + return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); + } + } +} diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs new file mode 100644 index 0000000..c06fc8d --- /dev/null +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -0,0 +1,195 @@ +using DistTestCore.Codex; +using KubernetesWorkflow; +using System.Globalization; + +namespace DistTestCore.Metrics +{ + public class MetricsQuery + { + private readonly Http http; + + public MetricsQuery(RunningContainers runningContainers) + { + RunningContainers = runningContainers; + + http = new Http( + runningContainers.RunningPod.Cluster.IP, + runningContainers.Containers[0].ServicePorts[0].Number, + "api/v1"); + } + + public RunningContainers RunningContainers { get; } + + public Metrics? GetMostRecent(string metricName, RunningContainer node) + { + var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); + if (response == null) return null; + + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapSingleValue(r.value) + }; + }).ToArray() + }; + } + + public Metrics? GetMetrics(string metricName) + { + var response = GetAll(metricName); + if (response == null) return null; + return MapResponseToMetrics(response); + } + + public Metrics? GetAllMetricsForNode(RunningContainer node) + { + var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return MapResponseToMetrics(response); + } + + private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); + if (response.status != "success") return null; + return response; + } + + private PrometheusQueryResponse? GetAll(string metricName) + { + var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return response; + } + + private Metrics MapResponseToMetrics(PrometheusQueryResponse response) + { + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Name = r.metric.__name__, + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } + + private MetricsSetValue[] MapSingleValue(object[] value) + { + if (value != null && value.Length > 0) + { + return new[] + { + MapValue(value) + }; + } + return Array.Empty(); + } + + private MetricsSetValue[] MapMultipleValues(object[][] values) + { + if (values != null && values.Length > 0) + { + return values.Select(v => MapValue(v)).ToArray(); + } + return Array.Empty(); + } + + private MetricsSetValue MapValue(object[] value) + { + if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + + return new MetricsSetValue + { + Timestamp = ToTimestamp(value[0]), + Value = ToValue(value[1]) + }; + } + + private string GetInstanceNameForNode(RunningContainer node) + { + var ip = node.Pod.Ip; + var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; + return $"{ip}:{port}"; + } + + private string GetInstanceStringForNode(RunningContainer node) + { + return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; + } + + private string GetQueryTimeRange() + { + return "[12h]"; + } + + private double ToValue(object v) + { + return Convert.ToDouble(v, CultureInfo.InvariantCulture); + } + + private DateTime ToTimestamp(object v) + { + var unixSeconds = ToValue(v); + return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); + } + } + + public class Metrics + { + public MetricsSet[] Sets { get; set; } = Array.Empty(); + } + + public class MetricsSet + { + public string Name { get; set; } = string.Empty; + public string Instance { get; set; } = string.Empty; + public MetricsSetValue[] Values { get; set; } = Array.Empty(); + } + + public class MetricsSetValue + { + public DateTime Timestamp { get; set; } + public double Value { get; set; } + } + + public class PrometheusQueryResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryResponseData data { get; set; } = new(); + } + + public class PrometheusQueryResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } + + public class PrometheusQueryResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public object[] value { get; set; } = Array.Empty(); + public object[][] values { get; set; } = Array.Empty(); + } + + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } + + public class PrometheusAllNamesResponse + { + public string status { get; set; } = string.Empty; + public string[] data { get; set; } = Array.Empty(); + } +} diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs new file mode 100644 index 0000000..5152ff7 --- /dev/null +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -0,0 +1,17 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Metrics +{ + public class PrometheusContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/prometheus-envconf:latest"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + AddExposedPortAndVar("PROM_PORT"); + AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); + } + } +} diff --git a/DistTestCore/Metrics/PrometheusStartupConfig.cs b/DistTestCore/Metrics/PrometheusStartupConfig.cs new file mode 100644 index 0000000..7bf7fe6 --- /dev/null +++ b/DistTestCore/Metrics/PrometheusStartupConfig.cs @@ -0,0 +1,12 @@ +namespace DistTestCore.Metrics +{ + public class PrometheusStartupConfig + { + public PrometheusStartupConfig(string prometheusConfigBase64) + { + PrometheusConfigBase64 = prometheusConfigBase64; + } + + public string PrometheusConfigBase64 { get; } + } +} diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 8f85f91..0efa51a 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Metrics; using NUnit.Framework; namespace DistTestCore @@ -11,7 +12,7 @@ namespace DistTestCore TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); ICodexNodeLog DownloadLog(); - //IMetricsAccess Metrics { get; } + IMetricsAccess Metrics { get; } //IMarketplaceAccess Marketplace { get; } } @@ -21,15 +22,17 @@ namespace DistTestCore private const string UploadFailedMessage = "Unable to store block"; private readonly TestLifecycle lifecycle; - public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group) + public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess) { this.lifecycle = lifecycle; CodexAccess = codexAccess; Group = group; + Metrics = metricsAccess; } public CodexAccess CodexAccess { get; } public CodexNodeGroup Group { get; } + public IMetricsAccess Metrics { get; } public string GetName() { diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs new file mode 100644 index 0000000..a58b3e4 --- /dev/null +++ b/DistTestCore/PrometheusStarter.cs @@ -0,0 +1,65 @@ +using DistTestCore.Codex; +using DistTestCore.Metrics; +using KubernetesWorkflow; +using System.Text; + +namespace DistTestCore +{ + public class PrometheusStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers) + { + if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); + + Log($"Starting metrics server for {containers.Describe()}"); + var startupConfig = new StartupConfig(); + startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); + + var workflow = workflowCreator.CreateWorkflow(); + var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); + if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); + + Log("Metrics server started."); + + return new CodexNodeMetricsAccessFactory(runningContainers); + } + + private string GeneratePrometheusConfig(RunningContainer[] nodes) + { + var config = ""; + config += "global:\n"; + config += " scrape_interval: 30s\n"; + config += " scrape_timeout: 10s\n"; + config += "\n"; + config += "scrape_configs:\n"; + config += " - job_name: services\n"; + config += " metrics_path: /metrics\n"; + config += " static_configs:\n"; + config += " - targets:\n"; + + foreach (var node in nodes) + { + var ip = node.Pod.Ip; + var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; + config += $" - '{ip}:{port}'\n"; + } + + var bytes = Encoding.ASCII.GetBytes(config); + return Convert.ToBase64String(bytes); + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 1267216..a5127a7 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,20 +1,27 @@ using DistTestCore.CodexLogsAndMetrics; +using KubernetesWorkflow; using Logging; namespace DistTestCore { public class TestLifecycle { + private readonly WorkflowCreator workflowCreator; + public TestLifecycle(Configuration configuration) { Log = new TestLog(configuration.GetLogConfig()); + workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + FileManager = new FileManager(Log, configuration); - CodexStarter = new CodexStarter(this, configuration); + CodexStarter = new CodexStarter(this, workflowCreator); + PrometheusStarter = new PrometheusStarter(this, workflowCreator); } public TestLog Log { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } + public PrometheusStarter PrometheusStarter { get; } public void DeleteAllResources() { diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index 7e8d90e..f676c7f 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -17,16 +17,23 @@ public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } public EnvVar[] EnvVars { get; } + + public Port GetPortByTag(string tag) + { + return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag); + } } public class Port { - public Port(int number) + public Port(int number, string tag) { Number = number; + Tag = tag; } public int Number { get; } + public string Tag { get; } } public class EnvVar diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 60ea91e..6c6a3ee 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -28,28 +28,28 @@ protected int ContainerNumber { get; private set; } = 0; protected abstract void Initialize(StartupConfig config); - protected Port AddExposedPort() + protected Port AddExposedPort(string tag = "") { - var p = factory.CreatePort(); + var p = factory.CreatePort(tag); exposedPorts.Add(p); return p; } - protected Port AddInternalPort() + protected Port AddInternalPort(string tag = "") { - var p = factory.CreatePort(); + var p = factory.CreatePort(tag); internalPorts.Add(p); return p; } - protected void AddExposedPortAndVar(string name) + protected void AddExposedPortAndVar(string name, string tag = "") { - AddEnvVar(name, AddExposedPort()); + AddEnvVar(name, AddExposedPort(tag)); } - protected void AddInternalPortAndVar(string name) + protected void AddInternalPortAndVar(string name, string tag = "") { - AddEnvVar(name, AddInternalPort()); + AddEnvVar(name, AddInternalPort(tag)); } protected void AddEnvVar(string name, string value) diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index eb492a9..394f86e 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -277,7 +277,7 @@ namespace KubernetesWorkflow foreach (var port in recipe.ExposedPorts) { var servicePort = workflowNumberSource.GetServicePort(); - usedPorts.Add(new Port(servicePort)); + usedPorts.Add(new Port(servicePort, "")); result.Add(new V1ServicePort { diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs index f99f345..cf1f67b 100644 --- a/KubernetesWorkflow/RecipeComponentFactory.cs +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -7,9 +7,9 @@ namespace KubernetesWorkflow { private NumberSource portNumberSource = new NumberSource(8080); - public Port CreatePort() + public Port CreatePort(string tag) { - return new Port(portNumberSource.GetNextNumber()); + return new Port(portNumberSource.GetNextNumber(), tag); } public EnvVar CreateEnvVar(string name, int value) diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 49fc65f..783c6f8 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -12,6 +12,11 @@ public StartupConfig StartupConfig { get; } public RunningPod RunningPod { get; } public RunningContainer[] Containers { get; } + + public string Describe() + { + return $"[{RunningPod.Ip}]"; + } } public class RunningContainer diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 6f6ce14..bf0c331 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -78,30 +78,30 @@ namespace Tests.BasicTests log.AssertLogContains("Uploaded file"); } - //[Test] - //public void TwoMetricsExample() - //{ - // var group = SetupCodexNodes(2) - // .EnableMetrics() - // .BringOnline(); + [Test] + public void TwoMetricsExample() + { + var group = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); - // var group2 = SetupCodexNodes(2) - // .EnableMetrics() - // .BringOnline(); + var group2 = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); - // var primary = group[0]; - // var secondary = group[1]; - // var primary2 = group2[0]; - // var secondary2 = group2[1]; + var primary = group[0]; + var secondary = group[1]; + var primary2 = group2[0]; + var secondary2 = group2[1]; - // primary.ConnectToPeer(secondary); - // primary2.ConnectToPeer(secondary2); + primary.ConnectToPeer(secondary); + primary2.ConnectToPeer(secondary2); - // Thread.Sleep(TimeSpan.FromMinutes(5)); + Thread.Sleep(TimeSpan.FromMinutes(5)); - // primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - // primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - //} + primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + } //[Test] //public void MarketplaceExample() From 9a458832784c029b2e31c356a600c45a00a7e5f9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 15:02:51 +0200 Subject: [PATCH 12/21] Restores downloading of metrics on test failure --- DistTestCore/DistTest.cs | 26 ++++++++++++++++++++--- DistTestCore/Metrics/MetricsAccess.cs | 5 +++++ DistTestCore/Metrics/MetricsDownloader.cs | 26 ++++------------------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index d1830ba..0cfea37 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,5 @@ using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Metrics; using NUnit.Framework; namespace DistTestCore @@ -76,7 +77,7 @@ namespace DistTestCore { Log("Downloading all CodexNode logs and metrics because of test failure..."); DownloadAllLogs(); - //k8sManager.DownloadAllMetrics(); + DownloadAllMetrics(); } else { @@ -102,10 +103,29 @@ namespace DistTestCore private void DownloadAllLogs() { - var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); - foreach (var node in allNodes) + OnEachCodexNode(node => { lifecycle.DownloadLog(node); + }); + } + + private void DownloadAllMetrics() + { + var metricsDownloader = new MetricsDownloader(lifecycle.Log); + + OnEachCodexNode(node => + { + var m = (MetricsAccess)node.Metrics; + metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m); + }); + } + + private void OnEachCodexNode(Action action) + { + var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); + foreach (var node in allNodes) + { + action(node); } } diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index 3287ea7..e5bd2b3 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -37,6 +37,11 @@ namespace DistTestCore.Metrics Assert.That(metricValue, constraint, message); } + public Metrics? GetAllMetrics() + { + return query.GetAllMetricsForNode(node); + } + private MetricsSet GetMetricWithTimeout(string metricName) { var start = DateTime.UtcNow; diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs index 3d79752..1ea56f3 100644 --- a/DistTestCore/Metrics/MetricsDownloader.cs +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -6,39 +6,21 @@ namespace DistTestCore.Metrics public class MetricsDownloader { private readonly TestLog log; - private readonly Dictionary activePrometheuses; - public MetricsDownloader(TestLog log, Dictionary activePrometheuses) + public MetricsDownloader(TestLog log) { this.log = log; - this.activePrometheuses = activePrometheuses; } - public void DownloadAllMetrics() + public void DownloadAllMetricsForNode(string nodeName, MetricsAccess access) { - foreach (var pair in activePrometheuses) - { - DownloadAllMetrics(pair.Key, pair.Value); - } - } - - private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) - { - foreach (var node in nodes) - { - DownloadAllMetricsForNode(query, node); - } - } - - private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) - { - var metrics = query.GetAllMetricsForNode(node.CodexAccess.Container); + var metrics = access.GetAllMetrics(); if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); var map = CreateValueMap(metrics); - WriteToFile(node.GetName(), headers, map); + WriteToFile(nodeName, headers, map); } private void WriteToFile(string nodeName, string[] headers, Dictionary> map) From 07fbda3f9a39002426f53e1533c9328f28ead453 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 15:04:01 +0200 Subject: [PATCH 13/21] Allows for metrics collection when some codex groups don't have metrics enabled. --- DistTestCore/DistTest.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 0cfea37..a088e7c 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -115,8 +115,11 @@ namespace DistTestCore OnEachCodexNode(node => { - var m = (MetricsAccess)node.Metrics; - metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m); + var m = node.Metrics as MetricsAccess; + if (m != null) + { + metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m); + } }); } From 4fd00607dfd42526bee153456376198e4b4b438e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 09:54:07 +0200 Subject: [PATCH 14/21] Setting up Geth starters --- CodexDistTestCore/K8sOperations.cs | 42 +------------- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 4 +- DistTestCore/CodexStarter.cs | 2 + DistTestCore/DistTest.cs | 2 +- DistTestCore/GethStarter.cs | 44 ++++++++++++++ .../CodexNodeLog.cs | 2 +- ...ownloadLogsAndMetricsOnFailureAttribute.cs | 2 +- .../LogDownloadHandler.cs | 2 +- .../Marketplace/GethBootstrapNodeInfo.cs | 18 ++++++ .../Marketplace/GethBootstrapNodeStarter.cs | 47 +++++++++++++++ .../Marketplace/GethCompanionNodeInfo.cs | 16 +++++ .../Marketplace/GethCompanionNodeStarter.cs | 50 ++++++++++++++++ .../Marketplace/GethContainerRecipe.cs | 37 ++++++++++++ DistTestCore/Marketplace/GethInfoExtractor.cs | 58 +++++++++++++++++++ DistTestCore/Marketplace/GethStartupConfig.cs | 14 +++++ DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/TestLifecycle.cs | 4 +- KubernetesWorkflow/CommandRunner.cs | 52 +++++++++++++++++ KubernetesWorkflow/K8sController.cs | 7 +++ KubernetesWorkflow/StartupWorkflow.cs | 8 +++ 20 files changed, 363 insertions(+), 50 deletions(-) create mode 100644 DistTestCore/GethStarter.cs rename DistTestCore/{CodexLogsAndMetrics => Logs}/CodexNodeLog.cs (95%) rename DistTestCore/{CodexLogsAndMetrics => Logs}/DontDownloadLogsAndMetricsOnFailureAttribute.cs (90%) rename DistTestCore/{CodexLogsAndMetrics => Logs}/LogDownloadHandler.cs (95%) create mode 100644 DistTestCore/Marketplace/GethBootstrapNodeInfo.cs create mode 100644 DistTestCore/Marketplace/GethBootstrapNodeStarter.cs create mode 100644 DistTestCore/Marketplace/GethCompanionNodeInfo.cs create mode 100644 DistTestCore/Marketplace/GethCompanionNodeStarter.cs create mode 100644 DistTestCore/Marketplace/GethContainerRecipe.cs create mode 100644 DistTestCore/Marketplace/GethInfoExtractor.cs create mode 100644 DistTestCore/Marketplace/GethStartupConfig.cs create mode 100644 KubernetesWorkflow/CommandRunner.cs diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 73f04b0..0cc3ea1 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -343,47 +343,7 @@ namespace CodexDistTestCore private class CommandRunner { - private readonly Kubernetes client; - private readonly PodInfo pod; - private readonly string containerName; - private readonly string command; - private readonly string[] arguments; - private readonly List lines = new List(); - - public CommandRunner(Kubernetes client, PodInfo pod, string containerName, string command, string[] arguments) - { - this.client = client; - this.pod = pod; - this.containerName = containerName; - this.command = command; - this.arguments = arguments; - } - - public void Run() - { - var input = new[] { command }.Concat(arguments).ToArray(); - - Utils.Wait(client.NamespacedPodExecAsync( - pod.Name, K8sCluster.K8sNamespace, containerName, input, false, Callback, new CancellationToken())); - } - - public string GetStdOut() - { - return string.Join(Environment.NewLine, lines); - } - - private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) - { - using var streamReader = new StreamReader(stdOut); - var line = streamReader.ReadLine(); - while (line != null) - { - lines.Add(line); - line = streamReader.ReadLine(); - } - - return Task.CompletedTask; - } + } } } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index f99d520..bfaf3d4 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -6,15 +6,13 @@ namespace CodexDistTestCore.Marketplace public static class GethDockerImage { public const string Image = "thatbenbierens/geth-confenv:latest"; - public const string AccountFilename = "account_string.txt"; - public const string GenesisFilename = "genesis.json"; + } public class K8sGethBoostrapSpecs { public const string ContainerName = "dtest-gethb"; private const string portName = "gethb"; - private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; public K8sGethBoostrapSpecs(int servicePort) { diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 65c9a32..b85fba4 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -18,6 +18,8 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { + var something = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); + var containers = StartCodexContainers(codexSetup); var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index a088e7c..e3631f4 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using DistTestCore.Metrics; using NUnit.Framework; diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs new file mode 100644 index 0000000..bf26523 --- /dev/null +++ b/DistTestCore/GethStarter.cs @@ -0,0 +1,44 @@ +using DistTestCore.Marketplace; +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class GethStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private GethBootstrapNodeInfo? bootstrapNode; + + public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + + bootstrapNodeStarter = new GethBootstrapNodeStarter(lifecycle, workflowCreator); + } + + public object BringOnlineMarketplaceFor(CodexSetup codexSetup) + { + EnsureBootstrapNode(); + StartCompanionNodes(codexSetup); + return null!; + } + + private void EnsureBootstrapNode() + { + if (bootstrapNode != null) return; + bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); + } + + private void StartCompanionNodes(CodexSetup codexSetup) + { + throw new NotImplementedException(); + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs b/DistTestCore/Logs/CodexNodeLog.cs similarity index 95% rename from DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs rename to DistTestCore/Logs/CodexNodeLog.cs index a4a9cb0..ac92678 100644 --- a/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs +++ b/DistTestCore/Logs/CodexNodeLog.cs @@ -1,7 +1,7 @@ using Logging; using NUnit.Framework; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { public interface ICodexNodeLog { diff --git a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs b/DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs similarity index 90% rename from DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs rename to DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs index 0cf1cbe..b95d875 100644 --- a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs +++ b/DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute diff --git a/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs similarity index 95% rename from DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs rename to DistTestCore/Logs/LogDownloadHandler.cs index 59c14d8..b6136c3 100644 --- a/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -1,7 +1,7 @@ using KubernetesWorkflow; using Logging; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { public class LogDownloadHandler : ILogHandler { diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs new file mode 100644 index 0000000..8f897ef --- /dev/null +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -0,0 +1,18 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethBootstrapNodeInfo + { + public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64) + { + RunningContainers = runningContainers; + Account = account; + GenesisJsonBase64 = genesisJsonBase64; + } + + public RunningContainers RunningContainers { get; } + public string Account { get; } + public string GenesisJsonBase64 { get; } + } +} diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs new file mode 100644 index 0000000..7c114c4 --- /dev/null +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -0,0 +1,47 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethBootstrapNodeStarter + { + private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public GethBootstrapNodeInfo StartGethBootstrapNode() + { + Log("Starting Geth bootstrap node..."); + var startupConfig = CreateBootstrapStartupConfig(); + + var workflow = workflowCreator.CreateWorkflow(); + var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); + if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); + + var extractor = new GethInfoExtractor(workflow, containers.Containers[0]); + var account = extractor.ExtractAccount(); + var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); + + Log($"Geth bootstrap node started with account '{account}'"); + + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64); + } + + private StartupConfig CreateBootstrapStartupConfig() + { + var config = new StartupConfig(); + config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64)); + return config; + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs new file mode 100644 index 0000000..9b7bd23 --- /dev/null +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -0,0 +1,16 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethCompanionNodeInfo + { + public GethCompanionNodeInfo(RunningContainer runningContainer, string account) + { + RunningContainer = runningContainer; + Account = account; + } + + public RunningContainer RunningContainer { get; } + public string Account { get; } + } +} diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs new file mode 100644 index 0000000..b8b178b --- /dev/null +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -0,0 +1,50 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethCompanionNodeStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) + { + Log($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes."); + + var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode); + + var workflow = workflowCreator.CreateWorkflow(); + var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig); + if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure."); + + Log("Initialized companion nodes."); + + return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray(); + } + + private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container) + { + var extractor = new GethInfoExtractor(workflow, container); + var account = extractor.ExtractAccount(); + return new GethCompanionNodeInfo(container, account); + } + + private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) + { + var config = new StartupConfig(); + config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64)); + return config; + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs new file mode 100644 index 0000000..3df72f6 --- /dev/null +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -0,0 +1,37 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/geth-confenv:latest"; + public const string AccountFilename = "account_string.txt"; + public const string GenesisFilename = "genesis.json"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + var args = CreateArgs(config); + + AddEnvVar("GETH_ARGS", args); + AddEnvVar("GENESIS_JSON", config.GenesisJsonBase64); + } + + private string CreateArgs(GethStartupConfig config) + { + if (config.IsBootstrapNode) + { + AddEnvVar("IS_BOOTSTRAP", "1"); + var exposedPort = AddExposedPort(); + return $"--http.port {exposedPort.Number}"; + } + + var port = AddInternalPort(); + var discovery = AddInternalPort(); + var authRpc = AddInternalPort(); + var httpPort = AddInternalPort(); + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}"; + } + } +} diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/GethInfoExtractor.cs new file mode 100644 index 0000000..f27e396 --- /dev/null +++ b/DistTestCore/Marketplace/GethInfoExtractor.cs @@ -0,0 +1,58 @@ +using KubernetesWorkflow; +using System.Text; + +namespace DistTestCore.Marketplace +{ + public class GethInfoExtractor + { + private readonly StartupWorkflow workflow; + private readonly RunningContainer container; + + public GethInfoExtractor(StartupWorkflow workflow, RunningContainer container) + { + this.workflow = workflow; + this.container = container; + } + + public string ExtractAccount() + { + var account = Retry(FetchAccount); + + if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); + + return account; + } + + public string ExtractGenesisJsonBase64() + { + var genesisJson = Retry(FetchGenesisJson); + + if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure."); + + var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + + return encoded; + } + + private string Retry(Func fetch) + { + var result = fetch(); + if (string.IsNullOrEmpty(result)) + { + Thread.Sleep(TimeSpan.FromSeconds(5)); + result = fetch(); + } + return result; + } + + private string FetchGenesisJson() + { + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename); + } + + private string FetchAccount() + { + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); + } + } +} diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs new file mode 100644 index 0000000..60164bb --- /dev/null +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -0,0 +1,14 @@ +namespace DistTestCore.Marketplace +{ + public class GethStartupConfig + { + public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64) + { + IsBootstrapNode = isBootstrapNode; + GenesisJsonBase64 = genesisJsonBase64; + } + + public bool IsBootstrapNode { get; } + public string GenesisJsonBase64 { get; } + } +} diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 0efa51a..76ab592 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using DistTestCore.Metrics; using NUnit.Framework; diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index a5127a7..63d3c7a 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using KubernetesWorkflow; using Logging; @@ -16,12 +16,14 @@ namespace DistTestCore FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this, workflowCreator); PrometheusStarter = new PrometheusStarter(this, workflowCreator); + GethStarter = new GethStarter(this, workflowCreator); } public TestLog Log { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } + public GethStarter GethStarter { get; } public void DeleteAllResources() { diff --git a/KubernetesWorkflow/CommandRunner.cs b/KubernetesWorkflow/CommandRunner.cs new file mode 100644 index 0000000..a045500 --- /dev/null +++ b/KubernetesWorkflow/CommandRunner.cs @@ -0,0 +1,52 @@ +using k8s; +using Utils; + +namespace KubernetesWorkflow +{ + public class CommandRunner + { + private readonly Kubernetes client; + private readonly string k8sNamespace; + private readonly RunningPod pod; + private readonly string containerName; + private readonly string command; + private readonly string[] arguments; + private readonly List lines = new List(); + + public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments) + { + this.client = client; + this.k8sNamespace = k8sNamespace; + this.pod = pod; + this.containerName = containerName; + this.command = command; + this.arguments = arguments; + } + + public void Run() + { + var input = new[] { command }.Concat(arguments).ToArray(); + + Time.Wait(client.NamespacedPodExecAsync( + pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())); + } + + public string GetStdOut() + { + return string.Join(Environment.NewLine, lines); + } + + private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) + { + using var streamReader = new StreamReader(stdOut); + var line = streamReader.ReadLine(); + while (line != null) + { + lines.Add(line); + line = streamReader.ReadLine(); + } + + return Task.CompletedTask; + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 394f86e..0804e8f 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -49,6 +49,13 @@ namespace KubernetesWorkflow logHandler.Log(stream); } + public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args) + { + var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args); + runner.Run(); + return runner.GetStdOut(); + } + public void DeleteAllResources() { DeleteNamespace(); diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 9dff328..e78a5b2 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -42,6 +42,14 @@ }); } + public string ExecuteCommand(RunningContainer container, string command, params string[] args) + { + return K8s(controller => + { + return controller.ExecuteCommand(container.Pod, container.Recipe.Name, command, args); + }); + } + public void DeleteAllResources() { K8s(controller => From 3d908bab6c272f63a91291c7066e64df0203a16f Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 10:51:35 +0200 Subject: [PATCH 15/21] Setting up nethereum library --- .../Marketplace/MarketplaceController.cs | 135 ------------------ .../Marketplace/MarketplaceInitialConfig.cs | 8 -- DistTestCore/Codex/CodexContainerRecipe.cs | 17 ++- DistTestCore/Codex/CodexStartupConfig.cs | 5 +- DistTestCore/CodexNodeFactory.cs | 8 +- DistTestCore/CodexSetup.cs | 13 +- DistTestCore/CodexStarter.cs | 23 ++- DistTestCore/DistTestCore.csproj | 2 +- DistTestCore/GethStarter.cs | 39 +++-- .../Marketplace/GethContainerRecipe.cs | 3 +- DistTestCore/Marketplace/GethStartResult.cs | 16 +++ DistTestCore/Marketplace/MarketplaceAccess.cs | 85 +++++++++++ .../Marketplace/MarketplaceAccessFactory.cs | 24 ++++ .../Marketplace/MarketplaceInitialConfig.cs | 12 ++ DistTestCore/OnlineCodexNode.cs | 7 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 4 +- KubernetesWorkflow/StartupWorkflow.cs | 2 +- Nethereum/NethereumWorkflow.cs | 41 ++++++ Nethereum/NethereumWorkflow.csproj | 19 +++ Nethereum/NethereumWorkflowCreator.cs | 29 ++++ cs-codex-dist-testing.sln | 14 +- 21 files changed, 318 insertions(+), 188 deletions(-) create mode 100644 DistTestCore/Marketplace/GethStartResult.cs create mode 100644 DistTestCore/Marketplace/MarketplaceAccess.cs create mode 100644 DistTestCore/Marketplace/MarketplaceAccessFactory.cs create mode 100644 DistTestCore/Marketplace/MarketplaceInitialConfig.cs create mode 100644 Nethereum/NethereumWorkflow.cs create mode 100644 Nethereum/NethereumWorkflow.csproj create mode 100644 Nethereum/NethereumWorkflowCreator.cs diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index 21bfc08..8bab9e9 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -1,7 +1,4 @@ using CodexDistTestCore.Config; -using Nethereum.Web3; -using Nethereum.Web3.Accounts; -using Nethereum.Web3.Accounts.Managed; using NUnit.Framework; using System.Numerics; using System.Text; @@ -22,138 +19,6 @@ namespace CodexDistTestCore.Marketplace this.k8sManager = k8sManager; } - public GethCompanionGroup BringOnlineMarketplace(OfflineCodexNodes offline) - { - if (bootstrapInfo == null) - { - BringOnlineBootstrapNode(); - } - log.Log($"Initializing companions for {offline.NumberOfNodes} Codex nodes."); - - var group = new GethCompanionGroup(companionGroupNumberSource.GetNextNumber(), CreateCompanionContainers(offline)); - group.Pod = k8sManager.BringOnlineGethCompanionGroup(bootstrapInfo!, group); - companionGroups.Add(group); - - log.Log("Initialized companion nodes."); - return group; - } - - private void BringOnlineBootstrapNode() - { - log.Log("Starting Geth bootstrap node..."); - var spec = k8sManager.CreateGethBootstrapNodeSpec(); - var pod = k8sManager.BringOnlineGethBootstrapNode(spec); - var (account, genesisJson) = ExtractAccountAndGenesisJson(pod); - bootstrapInfo = new GethBootstrapInfo(spec, pod, account, genesisJson); - log.Log($"Geth boothstrap node started."); - } - - private GethCompanionNodeContainer[] CreateCompanionContainers(OfflineCodexNodes offline) - { - var numberSource = new NumberSource(8080); - var result = new List(); - for (var i = 0; i < offline.NumberOfNodes; i++) result.Add(CreateGethNodeContainer(numberSource, i)); - return result.ToArray(); - } - - private GethCompanionNodeContainer CreateGethNodeContainer(NumberSource numberSource, int n) - { - return new GethCompanionNodeContainer( - name: $"geth-node{n}", - apiPort: numberSource.GetNextNumber(), - authRpcPort: numberSource.GetNextNumber(), - rpcPort: numberSource.GetNextNumber(), - containerPortName: $"geth-{n}" - ); - } - - private readonly K8sCluster k8sCluster = new K8sCluster(); - - public void AddToBalance(string account, decimal amount) - { - if (amount < 1 || string.IsNullOrEmpty(account)) Assert.Fail("Invalid arguments for AddToBalance"); - - // call the bootstrap node and convince it to give 'account' 'amount' tokens somehow. - - var web3 = CreateWeb3(); - - //var blockNumber1 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - //Thread.Sleep(TimeSpan.FromSeconds(5)); - //var blockNumber2 = Utils.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); - - //var bootstrapBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(bootstrapInfo.Account)); - - - var bigint = new BigInteger(amount); - var str = bigint.ToString("X"); - var value = new Nethereum.Hex.HexTypes.HexBigInteger(str); - var aaa = Utils.Wait(web3.Eth.TransactionManager.SendTransactionAsync(bootstrapInfo!.Account, account, value)); - var receipt = Utils.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(aaa)); - - //var receipt = Utils.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(account, amount)); - //var targetBalance = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); - } - - public decimal GetBalance(string account) - { - var web3 = CreateWeb3(); - var bigInt = Utils.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); - return (decimal)bigInt.Value; - } - - private Web3 CreateWeb3() - { - var ip = k8sCluster.GetIp(); - var port = bootstrapInfo!.Spec.ServicePort; - //var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); - return new Web3($"http://{ip}:{port}"); - } - - private (string, string) ExtractAccountAndGenesisJson(PodInfo pod) - { - var (account, genesisJson) = FetchAccountAndGenesisJson(pod); - if (string.IsNullOrEmpty(account) || string.IsNullOrEmpty(genesisJson)) - { - Thread.Sleep(TimeSpan.FromSeconds(15)); - (account, genesisJson) = FetchAccountAndGenesisJson(pod); - } - - Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); - Assert.That(genesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure."); - - var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); - - log.Log($"Initialized geth bootstrap node with account '{account}'"); - return (account, encoded); - } - - private (string, string) FetchAccountAndGenesisJson(PodInfo pod) - { - var bootstrapAccount = ExecuteCommand(pod, "cat", GethDockerImage.AccountFilename); - var bootstrapGenesisJson = ExecuteCommand(pod, "cat", GethDockerImage.GenesisFilename); - return (bootstrapAccount, bootstrapGenesisJson); - } - - private string ExecuteCommand(PodInfo pod, string command, params string[] arguments) - { - return k8sManager.ExecuteCommand(pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); - } - } - - public class GethBootstrapInfo - { - public GethBootstrapInfo(K8sGethBoostrapSpecs spec, PodInfo pod, string account, string genesisJsonBase64) - { - Spec = spec; - Pod = pod; - Account = account; - GenesisJsonBase64 = genesisJsonBase64; - } - - public K8sGethBoostrapSpecs Spec { get; } - public PodInfo Pod { get; } - public string Account { get; } - public string GenesisJsonBase64 { get; } } } diff --git a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs index f7761f5..23ad25f 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -1,12 +1,4 @@ namespace CodexDistTestCore.Marketplace { - public class MarketplaceInitialConfig - { - public MarketplaceInitialConfig(int initialBalance) - { - InitialBalance = initialBalance; - } - public int InitialBalance { get; } - } } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index e49dd84..31a9a7f 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Marketplace; +using KubernetesWorkflow; namespace DistTestCore.Codex { @@ -32,6 +33,20 @@ namespace DistTestCore.Codex AddEnvVar("METRICS_ADDR", "0.0.0.0"); AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag); } + + if (config.MarketplaceConfig != null) + { + var gethConfig = startupConfig.Get(); + var companionNode = gethConfig.CompanionNodes[Index]; + + // Bootstrap node access from within the cluster: + //var ip = gethConfig.BootstrapNode.RunningContainers.RunningPod.Ip; + //var port = gethConfig.BootstrapNode.RunningContainers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag); + + //AddEnvVar("ETH_PROVIDER", "todo"); + //AddEnvVar("ETH_ACCOUNT", companionNode.Account); + //AddEnvVar("ETH_DEPLOYMENT", "todo"); + } } } } diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index f83d629..915d4b9 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Marketplace; +using KubernetesWorkflow; namespace DistTestCore.Codex { @@ -8,8 +9,8 @@ namespace DistTestCore.Codex public CodexLogLevel? LogLevel { get; set; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } + public MarketplaceInitialConfig? MarketplaceConfig { get; set; } //public IOnlineCodexNode? BootstrapNode { get; private set; } - //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } } } diff --git a/DistTestCore/CodexNodeFactory.cs b/DistTestCore/CodexNodeFactory.cs index 3dce8aa..8320111 100644 --- a/DistTestCore/CodexNodeFactory.cs +++ b/DistTestCore/CodexNodeFactory.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Marketplace; using DistTestCore.Metrics; namespace DistTestCore @@ -12,17 +13,20 @@ namespace DistTestCore { private readonly TestLifecycle lifecycle; private readonly IMetricsAccessFactory metricsAccessFactory; + private readonly IMarketplaceAccessFactory marketplaceAccessFactory; - public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory) + public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory, IMarketplaceAccessFactory marketplaceAccessFactory) { this.lifecycle = lifecycle; this.metricsAccessFactory = metricsAccessFactory; + this.marketplaceAccessFactory = marketplaceAccessFactory; } public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group) { var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container); - return new OnlineCodexNode(lifecycle, access, group, metricsAccess); + var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(); + return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess); } } } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index ddc6ef0..5acedcb 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Marketplace; using KubernetesWorkflow; namespace DistTestCore @@ -10,7 +11,7 @@ namespace DistTestCore //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup EnableMetrics(); - //ICodexSetupConfig EnableMarketplace(int initialBalance); + ICodexSetup EnableMarketplace(int initialBalance); ICodexNodeGroup BringOnline(); } @@ -61,11 +62,11 @@ namespace DistTestCore return this; } - //public ICodexSetupConfig EnableMarketplace(int initialBalance) - //{ - // MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); - // return this; - //} + public ICodexSetup EnableMarketplace(int initialBalance) + { + MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); + return this; + } public string Describe() { diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index b85fba4..e6d4484 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -18,16 +18,21 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - var something = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); + Log($"Starting {codexSetup.Describe()}..."); + var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); - var containers = StartCodexContainers(codexSetup); + var startupConfig = new StartupConfig(); + startupConfig.Add(codexSetup); + startupConfig.Add(gethStartResult); + + var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); - var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory); + var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - + Log($"Started at '{group.Containers.RunningPod.Ip}'"); return group; } @@ -54,15 +59,10 @@ namespace DistTestCore workflow.DownloadContainerLog(container, logHandler); } - private RunningContainers StartCodexContainers(CodexSetup codexSetup) + private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location) { - Log($"Starting {codexSetup.Describe()}..."); - var workflow = CreateWorkflow(); - var startupConfig = new StartupConfig(); - startupConfig.Add(codexSetup); - - return workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); + return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig); } private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory) @@ -70,7 +70,6 @@ namespace DistTestCore var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); - Log($"Started at '{group.Containers.RunningPod.Ip}'"); return group; } diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index c57463d..944e748 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -8,7 +8,6 @@ - @@ -17,5 +16,6 @@ + diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index bf26523..c0db927 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -6,23 +6,43 @@ namespace DistTestCore public class GethStarter { private readonly TestLifecycle lifecycle; - private readonly WorkflowCreator workflowCreator; private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private readonly GethCompanionNodeStarter companionNodeStarter; private GethBootstrapNodeInfo? bootstrapNode; public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { this.lifecycle = lifecycle; - this.workflowCreator = workflowCreator; bootstrapNodeStarter = new GethBootstrapNodeStarter(lifecycle, workflowCreator); + companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); } - public object BringOnlineMarketplaceFor(CodexSetup codexSetup) + public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup) { + if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult(); + EnsureBootstrapNode(); - StartCompanionNodes(codexSetup); - return null!; + var companionNodes = StartCompanionNodes(codexSetup); + + TransferInitialBalance(codexSetup.MarketplaceConfig.InitialBalance, bootstrapNode, companionNodes); + + return new GethStartResult(CreateMarketplaceAccessFactory(), bootstrapNode!, companionNodes); + } + + private void TransferInitialBalance(int initialBalance, GethBootstrapNodeInfo? bootstrapNode, GethCompanionNodeInfo[] companionNodes) + { + aaaa + } + + private GethStartResult CreateMarketplaceUnavailableResult() + { + return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty()); + } + + private IMarketplaceAccessFactory CreateMarketplaceAccessFactory() + { + throw new NotImplementedException(); } private void EnsureBootstrapNode() @@ -31,14 +51,9 @@ namespace DistTestCore bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); } - private void StartCompanionNodes(CodexSetup codexSetup) + private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup) { - throw new NotImplementedException(); - } - - private void Log(string msg) - { - lifecycle.Log.Log(msg); + return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode!); } } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 3df72f6..2db8529 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -5,6 +5,7 @@ namespace DistTestCore.Marketplace public class GethContainerRecipe : ContainerRecipeFactory { protected override string Image => "thatbenbierens/geth-confenv:latest"; + public const string HttpPortTag = "http_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; @@ -30,7 +31,7 @@ namespace DistTestCore.Marketplace var port = AddInternalPort(); var discovery = AddInternalPort(); var authRpc = AddInternalPort(); - var httpPort = AddInternalPort(); + var httpPort = AddInternalPort(tag: HttpPortTag); return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}"; } } diff --git a/DistTestCore/Marketplace/GethStartResult.cs b/DistTestCore/Marketplace/GethStartResult.cs new file mode 100644 index 0000000..f1d9c97 --- /dev/null +++ b/DistTestCore/Marketplace/GethStartResult.cs @@ -0,0 +1,16 @@ +namespace DistTestCore.Marketplace +{ + public class GethStartResult + { + public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes) + { + MarketplaceAccessFactory = marketplaceAccessFactory; + BootstrapNode = bootstrapNode; + CompanionNodes = companionNodes; + } + + public IMarketplaceAccessFactory MarketplaceAccessFactory { get; } + public GethBootstrapNodeInfo BootstrapNode { get; } + public GethCompanionNodeInfo[] CompanionNodes { get; } + } +} diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs new file mode 100644 index 0000000..aa55cbf --- /dev/null +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -0,0 +1,85 @@ +using Logging; +using NUnit.Framework; +using NUnit.Framework.Constraints; + +namespace DistTestCore.Marketplace +{ + public interface IMarketplaceAccess + { + void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral); + void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes); + void AssertThatBalance(IResolveConstraint constraint, string message = ""); + decimal GetBalance(); + } + + public class MarketplaceAccess : IMarketplaceAccess + { + private readonly TestLog log; + private readonly CodexNodeGroup group; + + public MarketplaceAccess(TestLog log, CodexNodeGroup group) + { + this.log = log; + this.group = group; + } + + public void Initialize() + { + EnsureAccount(); + + marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); + + log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); + } + + public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) + { + throw new NotImplementedException(); + } + + public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) + { + throw new NotImplementedException(); + } + + public void AssertThatBalance(IResolveConstraint constraint, string message = "") + { + throw new NotImplementedException(); + } + + public decimal GetBalance() + { + return marketplaceController.GetBalance(container.Account); + } + } + + public class MarketplaceUnavailable : IMarketplaceAccess + { + public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) + { + Unavailable(); + } + + public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) + { + Unavailable(); + } + + public void AssertThatBalance(IResolveConstraint constraint, string message = "") + { + Unavailable(); + } + + public decimal GetBalance() + { + Unavailable(); + return 0; + } + + private void Unavailable() + { + Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } + } +} diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs new file mode 100644 index 0000000..e3de573 --- /dev/null +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -0,0 +1,24 @@ +namespace DistTestCore.Marketplace +{ + public interface IMarketplaceAccessFactory + { + IMarketplaceAccess CreateMarketplaceAccess(); + } + + public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory + { + public IMarketplaceAccess CreateMarketplaceAccess() + { + return new MarketplaceUnavailable(); + } + } + + public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory + { + public IMarketplaceAccess CreateMarketplaceAccess() + { + + return new MarketplaceAccess(query, codexContainer); + } + } +} diff --git a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs new file mode 100644 index 0000000..a815842 --- /dev/null +++ b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -0,0 +1,12 @@ +namespace DistTestCore.Marketplace +{ + public class MarketplaceInitialConfig + { + public MarketplaceInitialConfig(int initialBalance) + { + InitialBalance = initialBalance; + } + + public int InitialBalance { get; } + } +} diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 76ab592..7075ef3 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using DistTestCore.Logs; +using DistTestCore.Marketplace; using DistTestCore.Metrics; using NUnit.Framework; @@ -13,7 +14,7 @@ namespace DistTestCore void ConnectToPeer(IOnlineCodexNode node); ICodexNodeLog DownloadLog(); IMetricsAccess Metrics { get; } - //IMarketplaceAccess Marketplace { get; } + IMarketplaceAccess Marketplace { get; } } public class OnlineCodexNode : IOnlineCodexNode @@ -22,17 +23,19 @@ namespace DistTestCore private const string UploadFailedMessage = "Unable to store block"; private readonly TestLifecycle lifecycle; - public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess) + public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess) { this.lifecycle = lifecycle; CodexAccess = codexAccess; Group = group; Metrics = metricsAccess; + Marketplace = marketplaceAccess; } public CodexAccess CodexAccess { get; } public CodexNodeGroup Group { get; } public IMetricsAccess Metrics { get; } + public IMarketplaceAccess Marketplace { get; } public string GetName() { diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 6c6a3ee..136b209 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -7,10 +7,11 @@ private readonly List envVars = new List(); private RecipeComponentFactory factory = null!; - public ContainerRecipe CreateRecipe(int containerNumber, RecipeComponentFactory factory, StartupConfig config) + public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config) { this.factory = factory; ContainerNumber = containerNumber; + Index = index; Initialize(config); @@ -26,6 +27,7 @@ protected abstract string Image { get; } protected int ContainerNumber { get; private set; } = 0; + protected int Index { get; private set; } = 0; protected abstract void Initialize(StartupConfig config); protected Port AddExposedPort(string tag = "") diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index e78a5b2..f575b15 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -68,7 +68,7 @@ var result = new List(); for (var i = 0; i < numberOfContainers; i++) { - result.Add(recipeFactory.CreateRecipe(numberSource.GetContainerNumber(), componentFactory, startupConfig)); + result.Add(recipeFactory.CreateRecipe(i ,numberSource.GetContainerNumber(), componentFactory, startupConfig)); } return result.ToArray(); diff --git a/Nethereum/NethereumWorkflow.cs b/Nethereum/NethereumWorkflow.cs new file mode 100644 index 0000000..795ef27 --- /dev/null +++ b/Nethereum/NethereumWorkflow.cs @@ -0,0 +1,41 @@ +using Nethereum.Hex.HexTypes; +using Nethereum.Web3; +using System.Numerics; +using Utils; + +namespace NethereumWorkflow +{ + public class NethereumWorkflow + { + private readonly Web3 web3; + private readonly string rootAccount; + + internal NethereumWorkflow(Web3 web3, string rootAccount) + { + this.web3 = web3; + this.rootAccount = rootAccount; + } + + public void AddToBalance(string account, decimal amount) + { + if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance"); + + var value = ToHexBig(amount); + var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value)); + Time.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId)); + } + + public decimal GetBalance(string account) + { + var bigInt = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); + return (decimal)bigInt.Value; + } + + private HexBigInteger ToHexBig(decimal amount) + { + var bigint = new BigInteger(amount); + var str = bigint.ToString("X"); + return new HexBigInteger(str); + } + } +} diff --git a/Nethereum/NethereumWorkflow.csproj b/Nethereum/NethereumWorkflow.csproj new file mode 100644 index 0000000..99771c5 --- /dev/null +++ b/Nethereum/NethereumWorkflow.csproj @@ -0,0 +1,19 @@ + + + + net6.0 + NethereumWorkflow + enable + enable + + + + + + + + + + + + diff --git a/Nethereum/NethereumWorkflowCreator.cs b/Nethereum/NethereumWorkflowCreator.cs new file mode 100644 index 0000000..33ebf92 --- /dev/null +++ b/Nethereum/NethereumWorkflowCreator.cs @@ -0,0 +1,29 @@ +using Nethereum.Web3; + +namespace NethereumWorkflow +{ + public class NethereumWorkflowCreator + { + private readonly string ip; + private readonly int port; + private readonly string rootAccount; + + public NethereumWorkflowCreator(string ip, int port, string rootAccount) + { + this.ip = ip; + this.port = port; + this.rootAccount = rootAccount; + } + + public NethereumWorkflow CreateWorkflow() + { + return new NethereumWorkflow(CreateWeb3(), rootAccount); + } + + private Web3 CreateWeb3() + { + //var bootstrapaccount = new ManagedAccount(bootstrapInfo.Account, "qwerty!@#$%^"); + return new Web3($"http://{ip}:{port}"); + } + } +} diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index de9f58f..b47da0c 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -9,13 +9,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\Test EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj", "{957DE3B8-9571-450A-8609-B267DCA8727C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethereum", "Nethereum\Nethereum.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -51,6 +53,10 @@ Global {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Debug|Any CPU.Build.0 = Debug|Any CPU {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.ActiveCfg = Release|Any CPU {8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}.Release|Any CPU.Build.0 = Release|Any CPU + {D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 419ea1854fdc64b583dac8d04c41caf988b41af9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 12:37:05 +0200 Subject: [PATCH 16/21] Marketplace test passes --- CodexDistTestCore/CodexDistTestCore.csproj | 2 - DistTestCore/Codex/CodexContainerRecipe.cs | 4 +- DistTestCore/CodexNodeFactory.cs | 2 +- DistTestCore/CodexStarter.cs | 1 - DistTestCore/DistTest.cs | 13 +++- DistTestCore/DistTestCore.csproj | 2 +- DistTestCore/GethStarter.cs | 59 +++++++++++++------ .../Marketplace/GethBootstrapNodeInfo.cs | 11 ++++ .../Marketplace/GethContainerRecipe.cs | 4 +- DistTestCore/Marketplace/MarketplaceAccess.cs | 22 +++---- .../Marketplace/MarketplaceAccessFactory.cs | 30 ++++++++-- .../Metrics/PrometheusContainerRecipe.cs | 4 +- KubernetesWorkflow/ContainerRecipe.cs | 4 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 9 ++- ...eumWorkflow.cs => NethereumInteraction.cs} | 27 +++++++-- ...ator.cs => NethereumInteractionCreator.cs} | 13 ++-- Tests/BasicTests/SimpleTests.cs | 56 +++++++++--------- cs-codex-dist-testing.sln | 2 +- 18 files changed, 175 insertions(+), 90 deletions(-) rename Nethereum/{NethereumWorkflow.cs => NethereumInteraction.cs} (60%) rename Nethereum/{NethereumWorkflowCreator.cs => NethereumInteractionCreator.cs} (57%) diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj index 142eefb..cb1fb2e 100644 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ b/CodexDistTestCore/CodexDistTestCore.csproj @@ -8,8 +8,6 @@ - - diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 31a9a7f..50d6441 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -5,9 +5,10 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { + public const string DockerImage = "thatbenbierens/nim-codex:sha-b204837"; public const string MetricsPortTag = "metrics_port"; - protected override string Image => "thatbenbierens/nim-codex:sha-b204837"; + protected override string Image => DockerImage; protected override void Initialize(StartupConfig startupConfig) { @@ -38,6 +39,7 @@ namespace DistTestCore.Codex { var gethConfig = startupConfig.Get(); var companionNode = gethConfig.CompanionNodes[Index]; + Additional(companionNode); // Bootstrap node access from within the cluster: //var ip = gethConfig.BootstrapNode.RunningContainers.RunningPod.Ip; diff --git a/DistTestCore/CodexNodeFactory.cs b/DistTestCore/CodexNodeFactory.cs index 8320111..9b67158 100644 --- a/DistTestCore/CodexNodeFactory.cs +++ b/DistTestCore/CodexNodeFactory.cs @@ -25,7 +25,7 @@ namespace DistTestCore public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group) { var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container); - var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(); + var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access); return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess); } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index e6d4484..e90c0f0 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -69,7 +69,6 @@ namespace DistTestCore { var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); - return group; } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index e3631f4..e6c9ac0 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,5 +1,8 @@ -using DistTestCore.Logs; +using DistTestCore.Codex; +using DistTestCore.Logs; +using DistTestCore.Marketplace; using DistTestCore.Metrics; +using Logging; using NUnit.Framework; namespace DistTestCore @@ -8,6 +11,7 @@ namespace DistTestCore public abstract class DistTest { private TestLifecycle lifecycle = null!; + private TestLog log = null!; [OneTimeSetUp] public void GlobalSetup() @@ -26,7 +30,10 @@ namespace DistTestCore Error($"Global setup cleanup failed with: {ex}"); throw; } - Log("Global setup cleanup successful"); + log.Log("Global setup cleanup successful"); + log.Log($"Codex image: {CodexContainerRecipe.DockerImage}"); + log.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}"); + log.Log($"Geth image: {GethContainerRecipe.DockerImage}"); } [SetUp] @@ -38,6 +45,7 @@ namespace DistTestCore } else { + log.Log($"Run: {TestContext.CurrentContext.Test.Name}"); CreateNewTestLifecycle(); } } @@ -47,6 +55,7 @@ namespace DistTestCore { try { + log.Log($"{TestContext.CurrentContext.Test.Name} = {TestContext.CurrentContext.Result.Outcome.Status}"); lifecycle.Log.EndTest(); IncludeLogsAndMetricsOnTestFailure(); lifecycle.DeleteAllResources(); diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index 944e748..ccd5f66 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -16,6 +16,6 @@ - + diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index c0db927..3e01a5e 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -5,34 +5,41 @@ namespace DistTestCore { public class GethStarter { - private readonly TestLifecycle lifecycle; - private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private readonly GethBootstrapNodeCache bootstrapNodeCache; private readonly GethCompanionNodeStarter companionNodeStarter; - private GethBootstrapNodeInfo? bootstrapNode; + private readonly TestLifecycle lifecycle; public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { - this.lifecycle = lifecycle; - - bootstrapNodeStarter = new GethBootstrapNodeStarter(lifecycle, workflowCreator); + bootstrapNodeCache = new GethBootstrapNodeCache(new GethBootstrapNodeStarter(lifecycle, workflowCreator)); companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator); + this.lifecycle = lifecycle; } public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup) { if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult(); - EnsureBootstrapNode(); - var companionNodes = StartCompanionNodes(codexSetup); + var bootstrapNode = bootstrapNodeCache.Get(); + var companionNodes = StartCompanionNodes(codexSetup, bootstrapNode); - TransferInitialBalance(codexSetup.MarketplaceConfig.InitialBalance, bootstrapNode, companionNodes); + TransferInitialBalance(bootstrapNode, codexSetup.MarketplaceConfig.InitialBalance, companionNodes); - return new GethStartResult(CreateMarketplaceAccessFactory(), bootstrapNode!, companionNodes); + return CreateGethStartResult(bootstrapNode, companionNodes); } - private void TransferInitialBalance(int initialBalance, GethBootstrapNodeInfo? bootstrapNode, GethCompanionNodeInfo[] companionNodes) + private void TransferInitialBalance(GethBootstrapNodeInfo bootstrapNode, int initialBalance, GethCompanionNodeInfo[] companionNodes) { - aaaa + var interaction = bootstrapNode.StartInteraction(lifecycle.Log); + foreach (var node in companionNodes) + { + interaction.TransferTo(node.Account, initialBalance); + } + } + + private GethStartResult CreateGethStartResult(GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes) + { + return new GethStartResult(CreateMarketplaceAccessFactory(bootstrapNode), bootstrapNode, companionNodes); } private GethStartResult CreateMarketplaceUnavailableResult() @@ -40,20 +47,34 @@ namespace DistTestCore return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty()); } - private IMarketplaceAccessFactory CreateMarketplaceAccessFactory() + private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(GethBootstrapNodeInfo bootstrapNode) { - throw new NotImplementedException(); + return new GethMarketplaceAccessFactory(lifecycle.Log, bootstrapNode!); } - private void EnsureBootstrapNode() + private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) { - if (bootstrapNode != null) return; - bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); + return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode); + } + } + + public class GethBootstrapNodeCache + { + private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private GethBootstrapNodeInfo? bootstrapNode; + + public GethBootstrapNodeCache(GethBootstrapNodeStarter bootstrapNodeStarter) + { + this.bootstrapNodeStarter = bootstrapNodeStarter; } - private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup) + public GethBootstrapNodeInfo Get() { - return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode!); + if (bootstrapNode == null) + { + bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); + } + return bootstrapNode; } } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 8f897ef..39b6715 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -1,4 +1,6 @@ using KubernetesWorkflow; +using Logging; +using NethereumWorkflow; namespace DistTestCore.Marketplace { @@ -14,5 +16,14 @@ namespace DistTestCore.Marketplace public RunningContainers RunningContainers { get; } public string Account { get; } public string GenesisJsonBase64 { get; } + + public NethereumInteraction StartInteraction(TestLog log) + { + var ip = RunningContainers.RunningPod.Cluster.IP; + var port = RunningContainers.Containers[0].ServicePorts[0].Number; + + var creator = new NethereumInteractionCreator(log, ip, port, Account); + return creator.CreateWorkflow(); + } } } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 2db8529..2dbfbde 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -4,10 +4,12 @@ namespace DistTestCore.Marketplace { public class GethContainerRecipe : ContainerRecipeFactory { - protected override string Image => "thatbenbierens/geth-confenv:latest"; + public const string DockerImage = "thatbenbierens/geth-confenv:latest"; public const string HttpPortTag = "http_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; + + protected override string Image => DockerImage; protected override void Initialize(StartupConfig startupConfig) { diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index aa55cbf..8de63dd 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -15,21 +15,14 @@ namespace DistTestCore.Marketplace public class MarketplaceAccess : IMarketplaceAccess { private readonly TestLog log; - private readonly CodexNodeGroup group; + private readonly GethBootstrapNodeInfo bootstrapNode; + private readonly GethCompanionNodeInfo companionNode; - public MarketplaceAccess(TestLog log, CodexNodeGroup group) + public MarketplaceAccess(TestLog log, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo companionNode) { this.log = log; - this.group = group; - } - - public void Initialize() - { - EnsureAccount(); - - marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); - - log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); + this.bootstrapNode = bootstrapNode; + this.companionNode = companionNode; } public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) @@ -44,12 +37,13 @@ namespace DistTestCore.Marketplace public void AssertThatBalance(IResolveConstraint constraint, string message = "") { - throw new NotImplementedException(); + Assert.That(GetBalance(), constraint, message); } public decimal GetBalance() { - return marketplaceController.GetBalance(container.Account); + var interaction = bootstrapNode.StartInteraction(log); + return interaction.GetBalance(companionNode.Account); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs index e3de573..fba1155 100644 --- a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -1,13 +1,16 @@ -namespace DistTestCore.Marketplace +using DistTestCore.Codex; +using Logging; + +namespace DistTestCore.Marketplace { public interface IMarketplaceAccessFactory { - IMarketplaceAccess CreateMarketplaceAccess(); + IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access); } public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory { - public IMarketplaceAccess CreateMarketplaceAccess() + public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access) { return new MarketplaceUnavailable(); } @@ -15,10 +18,25 @@ public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory { - public IMarketplaceAccess CreateMarketplaceAccess() + private readonly TestLog log; + private readonly GethBootstrapNodeInfo bootstrapNode; + + public GethMarketplaceAccessFactory(TestLog log, GethBootstrapNodeInfo bootstrapNode) { - - return new MarketplaceAccess(query, codexContainer); + this.log = log; + this.bootstrapNode = bootstrapNode; + } + + public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access) + { + var companionNode = GetGethCompanionNode(access); + return new MarketplaceAccess(log, bootstrapNode, companionNode); + } + + private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access) + { + var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo); + return (GethCompanionNodeInfo)node; } } } diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs index 5152ff7..46587cf 100644 --- a/DistTestCore/Metrics/PrometheusContainerRecipe.cs +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -4,7 +4,9 @@ namespace DistTestCore.Metrics { public class PrometheusContainerRecipe : ContainerRecipeFactory { - protected override string Image => "thatbenbierens/prometheus-envconf:latest"; + public const string DockerImage = "thatbenbierens/prometheus-envconf:latest"; + + protected override string Image => DockerImage; protected override void Initialize(StartupConfig startupConfig) { diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index f676c7f..9fbbb9f 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -2,13 +2,14 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars) + public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, object[] additionals) { Number = number; Image = image; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; + Additionals = additionals; } public string Name { get { return $"ctnr{Number}"; } } @@ -17,6 +18,7 @@ public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } public EnvVar[] EnvVars { get; } + public object[] Additionals { get; } public Port GetPortByTag(string tag) { diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 136b209..07dc58e 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -5,6 +5,7 @@ private readonly List exposedPorts = new List(); private readonly List internalPorts = new List(); private readonly List envVars = new List(); + private readonly List additionals = new List(); private RecipeComponentFactory factory = null!; public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config) @@ -15,11 +16,12 @@ Initialize(config); - var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray()); + var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), additionals.ToArray()); exposedPorts.Clear(); internalPorts.Clear(); envVars.Clear(); + additionals.Clear(); this.factory = null!; return recipe; @@ -63,5 +65,10 @@ { envVars.Add(factory.CreateEnvVar(name, value.Number)); } + + protected void Additional(object userData) + { + additionals.Add(userData); + } } } diff --git a/Nethereum/NethereumWorkflow.cs b/Nethereum/NethereumInteraction.cs similarity index 60% rename from Nethereum/NethereumWorkflow.cs rename to Nethereum/NethereumInteraction.cs index 795ef27..832f32f 100644 --- a/Nethereum/NethereumWorkflow.cs +++ b/Nethereum/NethereumInteraction.cs @@ -1,34 +1,41 @@ -using Nethereum.Hex.HexTypes; +using Logging; +using Nethereum.Hex.HexTypes; using Nethereum.Web3; using System.Numerics; using Utils; namespace NethereumWorkflow { - public class NethereumWorkflow + public class NethereumInteraction { + private readonly TestLog log; private readonly Web3 web3; private readonly string rootAccount; - internal NethereumWorkflow(Web3 web3, string rootAccount) + internal NethereumInteraction(TestLog log, Web3 web3, string rootAccount) { + this.log = log; this.web3 = web3; this.rootAccount = rootAccount; } - public void AddToBalance(string account, decimal amount) + public void TransferTo(string account, decimal amount) { if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance"); var value = ToHexBig(amount); var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value)); Time.Wait(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId)); + + Log($"Transferred {amount} to {account}"); } public decimal GetBalance(string account) { var bigInt = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(account)); - return (decimal)bigInt.Value; + var result = ToDecimal(bigInt); + Log($"Balance of {account} is {result}"); + return result; } private HexBigInteger ToHexBig(decimal amount) @@ -37,5 +44,15 @@ namespace NethereumWorkflow var str = bigint.ToString("X"); return new HexBigInteger(str); } + + private decimal ToDecimal(HexBigInteger hexBigInteger) + { + return (decimal)hexBigInteger.Value; + } + + private void Log(string msg) + { + log.Log(msg); + } } } diff --git a/Nethereum/NethereumWorkflowCreator.cs b/Nethereum/NethereumInteractionCreator.cs similarity index 57% rename from Nethereum/NethereumWorkflowCreator.cs rename to Nethereum/NethereumInteractionCreator.cs index 33ebf92..b0b41f7 100644 --- a/Nethereum/NethereumWorkflowCreator.cs +++ b/Nethereum/NethereumInteractionCreator.cs @@ -1,23 +1,26 @@ -using Nethereum.Web3; +using Logging; +using Nethereum.Web3; namespace NethereumWorkflow { - public class NethereumWorkflowCreator + public class NethereumInteractionCreator { + private readonly TestLog log; private readonly string ip; private readonly int port; private readonly string rootAccount; - public NethereumWorkflowCreator(string ip, int port, string rootAccount) + public NethereumInteractionCreator(TestLog log, string ip, int port, string rootAccount) { + this.log = log; this.ip = ip; this.port = port; this.rootAccount = rootAccount; } - public NethereumWorkflow CreateWorkflow() + public NethereumInteraction CreateWorkflow() { - return new NethereumWorkflow(CreateWeb3(), rootAccount); + return new NethereumInteraction(log, CreateWeb3(), rootAccount); } private Web3 CreateWeb3() diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index bf0c331..943b598 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -103,42 +103,42 @@ namespace Tests.BasicTests primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); } - //[Test] - //public void MarketplaceExample() - //{ - // var group = SetupCodexNodes(4) - // .WithStorageQuota(10.GB()) - // .EnableMarketplace(initialBalance: 20) - // .BringOnline(); + [Test] + public void MarketplaceExample() + { + var group = SetupCodexNodes(4) + .WithStorageQuota(10.GB()) + .EnableMarketplace(initialBalance: 20) + .BringOnline(); - // foreach (var node in group) - // { - // Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); - // } + foreach (var node in group) + { + Assert.That(node.Marketplace.GetBalance(), Is.EqualTo(20)); + } - // // WIP: Balance is now only ETH. - // // todo: All nodes should have plenty of ETH to pay for transactions. - // // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. + // WIP: Balance is now only ETH. + // todo: All nodes should have plenty of ETH to pay for transactions. + // todo: Upload our own token, use this exclusively. ETH should be invisibile to the tests. - // //var secondary = SetupCodexNodes(1) - // // .EnableMarketplace(initialBalance: 1000) - // // .BringOnline()[0]; + //var secondary = SetupCodexNodes(1) + // .EnableMarketplace(initialBalance: 1000) + // .BringOnline()[0]; - // //primary.ConnectToPeer(secondary); - // //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); + //primary.ConnectToPeer(secondary); + //primary.Marketplace.MakeStorageAvailable(10.GB(), minPricePerBytePerSecond: 1, maxCollateral: 20); - // //var testFile = GenerateTestFile(10.MB()); - // //var contentId = secondary.UploadFile(testFile); - // //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, - // // requiredCollateral: 10, minRequiredNumberOfNodes: 1); + //var testFile = GenerateTestFile(10.MB()); + //var contentId = secondary.UploadFile(testFile); + //secondary.Marketplace.RequestStorage(contentId, pricePerBytePerSecond: 2, + // requiredCollateral: 10, minRequiredNumberOfNodes: 1); - // //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); - // //var primaryBalance = primary.Marketplace.GetBalance(); + //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); + //var primaryBalance = primary.Marketplace.GetBalance(); - // //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); - // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); - //} + //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); + //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + } private void PerformOneClientTest(IOnlineCodexNode primary) { diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index b47da0c..1f5c04f 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -17,7 +17,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Nethereum", "Nethereum\Nethereum.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From 60e653b63cd7ef3b8e52fce2a0bb03e9830c7a7a Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 14:53:39 +0200 Subject: [PATCH 17/21] Better logging + time measurement --- DistTestCore/CodexNodeGroup.cs | 2 +- DistTestCore/CodexSetup.cs | 2 +- DistTestCore/DistTest.cs | 65 ++++++++++++--- DistTestCore/Logs/CodexNodeLog.cs | 2 +- DistTestCore/Logs/LogDownloadHandler.cs | 2 +- DistTestCore/Metrics/MetricsDownloader.cs | 2 +- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/Stopwatch.cs | 19 +++++ DistTestCore/TestLifecycle.cs | 6 +- Logging/BaseLog.cs | 36 ++++++++ Logging/FixtureLog.cs | 49 +++++++++++ Logging/LogFile.cs | 42 +++------- Logging/TestLog.cs | 51 +++++------- .../{SimpleTests.cs => ExampleTests.cs} | 83 +------------------ Tests/BasicTests/OneClientTests.cs | 41 +++++++++ Tests/BasicTests/TwoClientTests.cs | 59 +++++++++++++ Utils/Time.cs | 10 +++ 17 files changed, 309 insertions(+), 164 deletions(-) create mode 100644 DistTestCore/Stopwatch.cs create mode 100644 Logging/BaseLog.cs create mode 100644 Logging/FixtureLog.cs rename Tests/BasicTests/{SimpleTests.cs => ExampleTests.cs} (56%) create mode 100644 Tests/BasicTests/OneClientTests.cs create mode 100644 Tests/BasicTests/TwoClientTests.cs diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index da4313f..25d46c1 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -73,7 +73,7 @@ namespace DistTestCore public string Describe() { - return $"CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}"; + return $""; } private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 5acedcb..612c37e 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -71,7 +71,7 @@ namespace DistTestCore public string Describe() { var args = string.Join(',', DescribeArgs()); - return $"{NumberOfNodes} CodexNodes with [{args}]"; + return $"({NumberOfNodes} CodexNodes with [{args}])"; } private IEnumerable DescribeArgs() diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index e6c9ac0..75711c8 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -2,27 +2,35 @@ using DistTestCore.Logs; using DistTestCore.Marketplace; using DistTestCore.Metrics; +using KubernetesWorkflow; using Logging; using NUnit.Framework; +using Utils; namespace DistTestCore { [SetUpFixture] public abstract class DistTest { + private readonly Configuration configuration = new Configuration(); + private FixtureLog fixtureLog = null!; private TestLifecycle lifecycle = null!; - private TestLog log = null!; + private DateTime testStart = DateTime.MinValue; [OneTimeSetUp] public void GlobalSetup() { // Previous test run may have been interrupted. // Begin by cleaning everything up. - CreateNewTestLifecycle(); + fixtureLog = new FixtureLog(configuration.GetLogConfig()); try { - lifecycle.DeleteAllResources(); + Stopwatch.Measure(fixtureLog, "Global setup", () => + { + var wc = new WorkflowCreator(configuration.GetK8sConfiguration()); + wc.CreateWorkflow().DeleteAllResources(); + }); } catch (Exception ex) { @@ -30,10 +38,11 @@ namespace DistTestCore Error($"Global setup cleanup failed with: {ex}"); throw; } - log.Log("Global setup cleanup successful"); - log.Log($"Codex image: {CodexContainerRecipe.DockerImage}"); - log.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}"); - log.Log($"Geth image: {GethContainerRecipe.DockerImage}"); + + fixtureLog.Log("Global setup cleanup successful"); + fixtureLog.Log($"Codex image: {CodexContainerRecipe.DockerImage}"); + fixtureLog.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}"); + fixtureLog.Log($"Geth image: {GethContainerRecipe.DockerImage}"); } [SetUp] @@ -45,7 +54,6 @@ namespace DistTestCore } else { - log.Log($"Run: {TestContext.CurrentContext.Test.Name}"); CreateNewTestLifecycle(); } } @@ -55,10 +63,7 @@ namespace DistTestCore { try { - log.Log($"{TestContext.CurrentContext.Test.Name} = {TestContext.CurrentContext.Result.Outcome.Status}"); - lifecycle.Log.EndTest(); - IncludeLogsAndMetricsOnTestFailure(); - lifecycle.DeleteAllResources(); + DisposeTestLifecycle(); } catch (Exception ex) { @@ -82,6 +87,8 @@ namespace DistTestCore var result = TestContext.CurrentContext.Result; if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { + fixtureLog.MarkAsFailed(); + if (IsDownloadingLogsAndMetricsEnabled()) { Log("Downloading all CodexNode logs and metrics because of test failure..."); @@ -107,7 +114,29 @@ namespace DistTestCore private void CreateNewTestLifecycle() { - lifecycle = new TestLifecycle(new Configuration()); + Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () => + { + lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration); + testStart = DateTime.UtcNow; + }); + } + + private void DisposeTestLifecycle() + { + fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})"); + Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => + { + lifecycle.Log.EndTest(); + IncludeLogsAndMetricsOnTestFailure(); + lifecycle.DeleteAllResources(); + lifecycle = null!; + }); + } + + private string GetTestDuration() + { + var testDuration = DateTime.UtcNow - testStart; + return Time.FormatDuration(testDuration); } private void DownloadAllLogs() @@ -141,6 +170,16 @@ namespace DistTestCore } } + private string GetCurrentTestName() + { + return $"[{TestContext.CurrentContext.Test.Name}]"; + } + + private string GetTestResult() + { + return TestContext.CurrentContext.Result.Outcome.Status.ToString(); + } + private bool IsDownloadingLogsAndMetricsEnabled() { var testProperties = TestContext.CurrentContext.Test.Properties; diff --git a/DistTestCore/Logs/CodexNodeLog.cs b/DistTestCore/Logs/CodexNodeLog.cs index ac92678..3b65241 100644 --- a/DistTestCore/Logs/CodexNodeLog.cs +++ b/DistTestCore/Logs/CodexNodeLog.cs @@ -29,7 +29,7 @@ namespace DistTestCore.Logs line = streamReader.ReadLine(); } - Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}"); + Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}"); } } } diff --git a/DistTestCore/Logs/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs index b6136c3..a01ad9e 100644 --- a/DistTestCore/Logs/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -21,7 +21,7 @@ namespace DistTestCore.Logs public void Log(Stream stream) { - log.Write($"{description} -->> {log.FilenameWithoutPath}"); + log.Write($"{description} -->> {log.FullFilename}"); log.WriteRaw(description); var reader = new StreamReader(stream); var line = reader.ReadLine(); diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs index 1ea56f3..4a458dd 100644 --- a/DistTestCore/Metrics/MetricsDownloader.cs +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -26,7 +26,7 @@ namespace DistTestCore.Metrics private void WriteToFile(string nodeName, string[] headers, Dictionary> map) { var file = log.CreateSubfile("csv"); - log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); + log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}"); file.WriteRaw(string.Join(",", headers)); diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 7075ef3..a12de9f 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -90,7 +90,7 @@ namespace DistTestCore public string Describe() { - return $"{Group.Describe()} contains {GetName()}"; + return $"({Group.Describe()} contains {GetName()})"; } private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) diff --git a/DistTestCore/Stopwatch.cs b/DistTestCore/Stopwatch.cs new file mode 100644 index 0000000..5be09c0 --- /dev/null +++ b/DistTestCore/Stopwatch.cs @@ -0,0 +1,19 @@ +using Logging; +using Utils; + +namespace DistTestCore +{ + public class Stopwatch + { + public static void Measure(BaseLog log, string name, Action action) + { + var start = DateTime.UtcNow; + + action(); + + var duration = DateTime.UtcNow - start; + + log.Log($"{name} ({Time.FormatDuration(duration)})"); + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 63d3c7a..2257610 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -8,9 +8,9 @@ namespace DistTestCore { private readonly WorkflowCreator workflowCreator; - public TestLifecycle(Configuration configuration) + public TestLifecycle(TestLog log, Configuration configuration) { - Log = new TestLog(configuration.GetLogConfig()); + Log = log; workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); FileManager = new FileManager(Log, configuration); @@ -37,7 +37,7 @@ namespace DistTestCore var description = node.Describe(); var handler = new LogDownloadHandler(description, subFile); - Log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); + Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'"); CodexStarter.DownloadLog(node.CodexAccess.Container, handler); return new CodexNodeLog(subFile); diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs new file mode 100644 index 0000000..e19ec12 --- /dev/null +++ b/Logging/BaseLog.cs @@ -0,0 +1,36 @@ +namespace Logging +{ + public abstract class BaseLog + { + private bool hasFailed; + private LogFile? logFile; + + protected abstract LogFile CreateLogFile(); + + protected LogFile LogFile + { + get + { + if (logFile == null) logFile = CreateLogFile(); + return logFile; + } + } + + public void Log(string message) + { + LogFile.Write(message); + } + + public void Error(string message) + { + Log($"[ERROR] {message}"); + } + + public void MarkAsFailed() + { + if (hasFailed) return; + hasFailed = true; + LogFile.ConcatToFilename("_FAILED"); + } + } +} diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs new file mode 100644 index 0000000..e06f4bc --- /dev/null +++ b/Logging/FixtureLog.cs @@ -0,0 +1,49 @@ +using NUnit.Framework; + +namespace Logging +{ + public class FixtureLog : BaseLog + { + private readonly DateTime start; + private readonly string fullName; + + public FixtureLog(LogConfig config) + { + start = DateTime.UtcNow; + var folder = DetermineFolder(config); // "root/2023-04 /14" + var fixtureName = GetFixtureName(); // "11-09-23Z_ExampleTests" + fullName = Path.Combine(folder, fixtureName); + } + + public TestLog CreateTestLog() + { + return new TestLog(fullName); + } + + protected override LogFile CreateLogFile() + { + return new LogFile(fullName, "log"); + } + + private string DetermineFolder(LogConfig config) + { + return Path.Join( + config.LogRoot, + $"{start.Year}-{Pad(start.Month)}", + Pad(start.Day)); + } + + private string GetFixtureName() + { + var test = TestContext.CurrentContext.Test; + var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); + return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}"; + } + + private static string Pad(int n) + { + return n.ToString().PadLeft(2, '0'); + } + + } +} diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs index 3a0063b..96ed16b 100644 --- a/Logging/LogFile.cs +++ b/Logging/LogFile.cs @@ -2,29 +2,19 @@ { public class LogFile { - private readonly DateTime now; - private string name; - private readonly string ext; - private readonly string filepath; + private readonly string extension; + private string filename; - public LogFile(LogConfig config, DateTime now, string name, string ext = "log") + public LogFile(string filename, string extension) { - this.now = now; - this.name = name; - this.ext = ext; + this.filename = filename; + this.extension = extension; + FullFilename = filename + "." + extension; - filepath = Path.Join( - config.LogRoot, - $"{now.Year}-{Pad(now.Month)}", - Pad(now.Day)); - - Directory.CreateDirectory(filepath); - - GenerateFilename(); + EnsurePathExists(filename); } - public string FullFilename { get; private set; } = string.Empty; - public string FilenameWithoutPath { get; private set; } = string.Empty; + public string FullFilename { get; private set; } public void Write(string message) { @@ -47,27 +37,21 @@ { var oldFullName = FullFilename; - name += toAdd; - - GenerateFilename(); + filename += toAdd; + FullFilename = filename + "." + extension; File.Move(oldFullName, FullFilename); } - private static string Pad(int n) - { - return n.ToString().PadLeft(2, '0'); - } - private static string GetTimestamp() { return $"[{DateTime.UtcNow.ToString("u")}]"; } - private void GenerateFilename() + private void EnsurePathExists(string filename) { - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; - FullFilename = Path.Combine(filepath, FilenameWithoutPath); + var path = new FileInfo(filename).Directory!.FullName; + Directory.CreateDirectory(path); } } } diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 7eb3979..8c4dc17 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -3,39 +3,30 @@ using Utils; namespace Logging { - public class TestLog + public class TestLog : BaseLog { private readonly NumberSource subfileNumberSource = new NumberSource(0); - private readonly LogFile file; - private readonly DateTime now; - private readonly LogConfig config; + private readonly string methodName; + private readonly string fullName; - public TestLog(LogConfig config) + public TestLog(string folder) { - this.config = config; - now = DateTime.UtcNow; + methodName = GetMethodName(); + fullName = Path.Combine(folder, methodName); - var name = GetTestName(); - file = new LogFile(config, now, name); - - Log($"Begin: {name}"); + Log($"Begin: {methodName}"); } - public void Log(string message) + public LogFile CreateSubfile(string ext = "log") { - file.Write(message); - } - - public void Error(string message) - { - Log($"[ERROR] {message}"); + return new LogFile($"{fullName}_{GetSubfileNumber()}", ext); } public void EndTest() { var result = TestContext.CurrentContext.Result; - Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); + Log($"Finished: {methodName} = {result.Outcome.Status}"); if (!string.IsNullOrEmpty(result.Message)) { Log(result.Message); @@ -44,26 +35,24 @@ namespace Logging if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { - RenameLogFile(); + MarkAsFailed(); } } - - private void RenameLogFile() + protected override LogFile CreateLogFile() { - file.ConcatToFilename("_FAILED"); + return new LogFile(fullName, "log"); } - public LogFile CreateSubfile(string ext = "log") - { - return new LogFile(config, now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); - } - - private static string GetTestName() + private string GetMethodName() { var test = TestContext.CurrentContext.Test; - var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); var args = FormatArguments(test); - return $"{className}.{test.MethodName}{args}"; + return $"{test.MethodName}{args}"; + } + + private string GetSubfileNumber() + { + return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0'); } private static string FormatArguments(TestContext.TestAdapter test) diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/ExampleTests.cs similarity index 56% rename from Tests/BasicTests/SimpleTests.cs rename to Tests/BasicTests/ExampleTests.cs index 943b598..a3e3ae4 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,69 +1,12 @@ using DistTestCore; using DistTestCore.Codex; -using KubernetesWorkflow; using NUnit.Framework; namespace Tests.BasicTests { [TestFixture] - public class SimpleTests : DistTest + public class ExampleTests : DistTest { - [Test] - public void OneClientTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - PerformOneClientTest(primary); - } - - [Test] - public void RestartTest() - { - var group = SetupCodexNodes(1).BringOnline(); - - var setup = group.BringOffline(); - - var primary = setup.BringOnline()[0]; - - PerformOneClientTest(primary); - } - - [Test] - public void TwoClientsOnePodTest() - { - var group = SetupCodexNodes(2).BringOnline(); - - var primary = group[0]; - var secondary = group[1]; - - PerformTwoClientTest(primary, secondary); - } - - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - var secondary = SetupCodexNodes(1).BringOnline()[0]; - - PerformTwoClientTest(primary, secondary); - } - - [Test] - [Ignore("Requires Location map to be configured for k8s cluster.")] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; - - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; - - PerformTwoClientTest(primary, secondary); - } - [Test] public void CodexLogExample() { @@ -139,29 +82,5 @@ namespace Tests.BasicTests //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); } - - private void PerformOneClientTest(IOnlineCodexNode primary) - { - var testFile = GenerateTestFile(1.MB()); - - var contentId = primary.UploadFile(testFile); - - var downloadedFile = primary.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - } - - private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) - { - primary.ConnectToPeer(secondary); - - var testFile = GenerateTestFile(1.MB()); - - var contentId = primary.UploadFile(testFile); - - var downloadedFile = secondary.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - } } } diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs new file mode 100644 index 0000000..76ff79b --- /dev/null +++ b/Tests/BasicTests/OneClientTests.cs @@ -0,0 +1,41 @@ +using DistTestCore; +using NUnit.Framework; + +namespace Tests.BasicTests +{ + [TestFixture] + public class OneClientTests : DistTest + { + [Test] + public void OneClientTest() + { + var primary = SetupCodexNodes(1).BringOnline()[0]; + + PerformOneClientTest(primary); + } + + [Test] + [Ignore("Unstable.")] + public void RestartTest() + { + var group = SetupCodexNodes(1).BringOnline(); + + var setup = group.BringOffline(); + + var primary = setup.BringOnline()[0]; + + PerformOneClientTest(primary); + } + + private void PerformOneClientTest(IOnlineCodexNode primary) + { + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = primary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs new file mode 100644 index 0000000..cde3b04 --- /dev/null +++ b/Tests/BasicTests/TwoClientTests.cs @@ -0,0 +1,59 @@ +using DistTestCore; +using KubernetesWorkflow; +using NUnit.Framework; + +namespace Tests.BasicTests +{ + [TestFixture] + public class TwoClientTests : DistTest + { + [Test] + public void TwoClientsOnePodTest() + { + var group = SetupCodexNodes(2).BringOnline(); + + var primary = group[0]; + var secondary = group[1]; + + PerformTwoClientTest(primary, secondary); + } + + [Test] + public void TwoClientsTwoPodsTest() + { + var primary = SetupCodexNodes(1).BringOnline()[0]; + + var secondary = SetupCodexNodes(1).BringOnline()[0]; + + PerformTwoClientTest(primary, secondary); + } + + [Test] + [Ignore("Requires Location map to be configured for k8s cluster.")] + public void TwoClientsTwoLocationsTest() + { + var primary = SetupCodexNodes(1) + .At(Location.BensLaptop) + .BringOnline()[0]; + + var secondary = SetupCodexNodes(1) + .At(Location.BensOldGamingMachine) + .BringOnline()[0]; + + PerformTwoClientTest(primary, secondary); + } + + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + primary.ConnectToPeer(secondary); + + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} diff --git a/Utils/Time.cs b/Utils/Time.cs index 98c5979..afe4f29 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -12,5 +12,15 @@ task.Wait(); return task.Result; } + + public static string FormatDuration(TimeSpan d) + { + var result = ""; + if (d.Days > 0) result += $"{d.Days} days, "; + if (d.Hours > 0) result += $"{d.Hours} hours, "; + if (d.Minutes > 0) result += $"{d.Minutes} mins, "; + result += $"{d.Seconds} secs"; + return result; + } } } From 3f159b8ece92fed6dfa56ef7860e7c463d011f58 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 07:56:08 +0200 Subject: [PATCH 18/21] Removes old backend --- CodexDistTestCore/ByteSize.cs | 57 --- CodexDistTestCore/CodexAPI.cs | 17 - CodexDistTestCore/CodexDistTestCore.csproj | 16 - CodexDistTestCore/CodexNodeContainer.cs | 89 ----- CodexDistTestCore/CodexNodeGroup.cs | 103 ------ CodexDistTestCore/CodexNodeLog.cs | 34 -- CodexDistTestCore/Config/CodexDockerImage.cs | 72 ---- CodexDistTestCore/Config/FileManagerConfig.cs | 7 - CodexDistTestCore/Config/K8sCluster.cs | 40 -- CodexDistTestCore/DistTest.cs | 120 ------ CodexDistTestCore/FileManager.cs | 110 ------ CodexDistTestCore/Http.cs | 100 ----- CodexDistTestCore/K8sManager.cs | 177 --------- CodexDistTestCore/K8sOperations.cs | 349 ------------------ CodexDistTestCore/KnownK8sPods.cs | 17 - .../Marketplace/GethCompanionNodeContainer.cs | 35 -- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 207 ----------- .../Marketplace/MarketplaceAccess.cs | 111 ------ .../Marketplace/MarketplaceController.cs | 24 -- .../Marketplace/MarketplaceInitialConfig.cs | 4 - .../Metrics/K8sPrometheusSpecs.cs | 122 ------ CodexDistTestCore/Metrics/MetricsAccess.cs | 63 ---- .../Metrics/MetricsAggregator.cs | 78 ---- .../Metrics/MetricsDownloader.cs | 97 ----- CodexDistTestCore/Metrics/MetricsQuery.cs | 190 ---------- CodexDistTestCore/NumberSource.cs | 19 - CodexDistTestCore/OfflineCodexNodes.cs | 97 ----- CodexDistTestCore/OnlineCodexNode.cs | 141 ------- CodexDistTestCore/PodLogDownloader.cs | 64 ---- CodexDistTestCore/Timing.cs | 131 ------- CodexDistTestCore/TryContract.cs | 101 ----- CodexDistTestCore/Utils.cs | 7 - LongTests/BasicTests/LargeFileTests.cs | 3 +- LongTests/BasicTests/TestInfraTests.cs | 3 +- LongTests/TestsLong.csproj | 2 +- cs-codex-dist-testing.sln | 8 +- 36 files changed, 6 insertions(+), 2809 deletions(-) delete mode 100644 CodexDistTestCore/ByteSize.cs delete mode 100644 CodexDistTestCore/CodexAPI.cs delete mode 100644 CodexDistTestCore/CodexDistTestCore.csproj delete mode 100644 CodexDistTestCore/CodexNodeContainer.cs delete mode 100644 CodexDistTestCore/CodexNodeGroup.cs delete mode 100644 CodexDistTestCore/CodexNodeLog.cs delete mode 100644 CodexDistTestCore/Config/CodexDockerImage.cs delete mode 100644 CodexDistTestCore/Config/FileManagerConfig.cs delete mode 100644 CodexDistTestCore/Config/K8sCluster.cs delete mode 100644 CodexDistTestCore/DistTest.cs delete mode 100644 CodexDistTestCore/FileManager.cs delete mode 100644 CodexDistTestCore/Http.cs delete mode 100644 CodexDistTestCore/K8sManager.cs delete mode 100644 CodexDistTestCore/K8sOperations.cs delete mode 100644 CodexDistTestCore/KnownK8sPods.cs delete mode 100644 CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs delete mode 100644 CodexDistTestCore/Marketplace/K8sGethSpecs.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceAccess.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceController.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs delete mode 100644 CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsAccess.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsAggregator.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsDownloader.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsQuery.cs delete mode 100644 CodexDistTestCore/NumberSource.cs delete mode 100644 CodexDistTestCore/OfflineCodexNodes.cs delete mode 100644 CodexDistTestCore/OnlineCodexNode.cs delete mode 100644 CodexDistTestCore/PodLogDownloader.cs delete mode 100644 CodexDistTestCore/Timing.cs delete mode 100644 CodexDistTestCore/TryContract.cs delete mode 100644 CodexDistTestCore/Utils.cs diff --git a/CodexDistTestCore/ByteSize.cs b/CodexDistTestCore/ByteSize.cs deleted file mode 100644 index e8f5c92..0000000 --- a/CodexDistTestCore/ByteSize.cs +++ /dev/null @@ -1,57 +0,0 @@ -namespace CodexDistTestCore -{ - public class ByteSize - { - public ByteSize(long sizeInBytes) - { - SizeInBytes = sizeInBytes; - } - - public long SizeInBytes { get; } - } - - public static class IntExtensions - { - private const long Kilo = 1024; - - public static ByteSize KB(this long i) - { - return new ByteSize(i * Kilo); - } - - public static ByteSize MB(this long i) - { - return (i * Kilo).KB(); - } - - public static ByteSize GB(this long i) - { - return (i * Kilo).MB(); - } - - public static ByteSize TB(this long i) - { - return (i * Kilo).GB(); - } - - public static ByteSize KB(this int i) - { - return Convert.ToInt64(i).KB(); - } - - public static ByteSize MB(this int i) - { - return Convert.ToInt64(i).MB(); - } - - public static ByteSize GB(this int i) - { - return Convert.ToInt64(i).GB(); - } - - public static ByteSize TB(this int i) - { - return Convert.ToInt64(i).TB(); - } - } -} diff --git a/CodexDistTestCore/CodexAPI.cs b/CodexDistTestCore/CodexAPI.cs deleted file mode 100644 index 997cd3c..0000000 --- a/CodexDistTestCore/CodexAPI.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace CodexDistTestCore -{ - public class CodexDebugResponse - { - public string id { get; set; } = string.Empty; - public string[] addrs { get; set; } = new string[0]; - public string repo { get; set; } = string.Empty; - public string spr { get; set; } = string.Empty; - public CodexDebugVersionResponse codex { get; set; } = new(); - } - - public class CodexDebugVersionResponse - { - public string version { get; set; } = string.Empty; - public string revision { get; set; } = string.Empty; - } -} diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj deleted file mode 100644 index cb1fb2e..0000000 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net6.0 - CodexDistTestCore - enable - enable - - - - - - - - - diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs deleted file mode 100644 index 71994ab..0000000 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ /dev/null @@ -1,89 +0,0 @@ -using CodexDistTestCore.Marketplace; - -namespace CodexDistTestCore -{ - public class CodexNodeContainer - { - public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort) - { - Name = name; - ServicePort = servicePort; - ServicePortName = servicePortName; - ApiPort = apiPort; - ContainerPortName = containerPortName; - DiscoveryPort = discoveryPort; - ListenPort = listenPort; - DataDir = dataDir; - MetricsPort = metricsPort; - } - - public string Name { get; } - public int ServicePort { get; } - public string ServicePortName { get; } - public int ApiPort { get; } - public string ContainerPortName { get; } - public int DiscoveryPort { get; } - public int ListenPort { get; } - public string DataDir { get; } - public int MetricsPort { get; } - - public GethCompanionNodeContainer? GethCompanionNodeContainer { get; set; } // :C - } - - public class CodexGroupNumberSource - { - private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0); - private readonly NumberSource groupContainerNameSource = new NumberSource(1); - private readonly NumberSource servicePortSource = new NumberSource(30001); - - public int GetNextCodexNodeGroupNumber() - { - return codexNodeGroupNumberSource.GetNextNumber(); - } - - public string GetNextServicePortName() - { - return $"node{groupContainerNameSource.GetNextNumber()}"; - } - - public int GetNextServicePort() - { - return servicePortSource.GetNextNumber(); - } - } - - public class CodexNodeContainerFactory - { - private readonly NumberSource containerNameSource = new NumberSource(1); - private readonly NumberSource codexPortSource = new NumberSource(8080); - private readonly CodexGroupNumberSource numberSource; - - public CodexNodeContainerFactory(CodexGroupNumberSource numberSource) - { - this.numberSource = numberSource; - } - - public CodexNodeContainer CreateNext(OfflineCodexNodes offline) - { - var n = containerNameSource.GetNextNumber(); - return new CodexNodeContainer( - name: $"codex-node{n}", - servicePort: numberSource.GetNextServicePort(), - servicePortName: numberSource.GetNextServicePortName(), - apiPort: codexPortSource.GetNextNumber(), - containerPortName: $"api-{n}", - discoveryPort: codexPortSource.GetNextNumber(), - listenPort: codexPortSource.GetNextNumber(), - dataDir: $"datadir{n}", - metricsPort: GetMetricsPort(offline) - ); - } - - private int GetMetricsPort(OfflineCodexNodes offline) - { - if (offline.MetricsEnabled) return codexPortSource.GetNextNumber(); - return 0; - } - - } -} diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs deleted file mode 100644 index 12031ac..0000000 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ /dev/null @@ -1,103 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using k8s.Models; -using System.Collections; - -namespace CodexDistTestCore -{ - public interface ICodexNodeGroup : IEnumerable - { - IOfflineCodexNodes BringOffline(); - IOnlineCodexNode this[int index] { get; } - } - - public class CodexNodeGroup : ICodexNodeGroup - { - private readonly TestLog log; - private readonly IK8sManager k8SManager; - - public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) - { - this.log = log; - OrderNumber = orderNumber; - Origin = origin; - this.k8SManager = k8SManager; - Nodes = nodes; - - foreach (var n in nodes) n.Group = this; - } - - public IOnlineCodexNode this[int index] - { - get - { - return Nodes[index]; - } - } - - public IOfflineCodexNodes BringOffline() - { - return k8SManager.BringOffline(this); - } - - public int OrderNumber { get; } - public OfflineCodexNodes Origin { get; } - public OnlineCodexNode[] Nodes { get; } - public V1Deployment? Deployment { get; set; } - public V1Service? Service { get; set; } - public PodInfo? PodInfo { get; set; } - public GethCompanionGroup? GethCompanionGroup { get; set; } - - public CodexNodeContainer[] GetContainers() - { - return Nodes.Select(n => n.Container).ToArray(); - } - - public IEnumerator GetEnumerator() - { - return Nodes.Cast().GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return Nodes.GetEnumerator(); - } - - public V1ObjectMeta GetServiceMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-entrypoint-" + OrderNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }; - } - - public V1ObjectMeta GetDeploymentMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-node-" + OrderNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }; - } - - public CodexNodeLog DownloadLog(IOnlineCodexNode node) - { - var logDownloader = new PodLogDownloader(log, k8SManager); - var n = (OnlineCodexNode)node; - return logDownloader.DownloadLog(n); - } - - public Dictionary GetSelector() - { - return new Dictionary { { "codex-test-node", "dist-test-" + OrderNumber } }; - } - - public string Describe() - { - return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}"; - } - } - - -} diff --git a/CodexDistTestCore/CodexNodeLog.cs b/CodexDistTestCore/CodexNodeLog.cs deleted file mode 100644 index 1a0572a..0000000 --- a/CodexDistTestCore/CodexNodeLog.cs +++ /dev/null @@ -1,34 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface ICodexNodeLog - { - void AssertLogContains(string expectedString); - } - - public class CodexNodeLog : ICodexNodeLog - { - private readonly LogFile logFile; - - public CodexNodeLog(LogFile logFile) - { - this.logFile = logFile; - } - - public void AssertLogContains(string expectedString) - { - using var file = File.OpenRead(logFile.FullFilename); - using var streamReader = new StreamReader(file); - - var line = streamReader.ReadLine(); - while (line != null) - { - if (line.Contains(expectedString)) return; - line = streamReader.ReadLine(); - } - - Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}"); - } - } -} diff --git a/CodexDistTestCore/Config/CodexDockerImage.cs b/CodexDistTestCore/Config/CodexDockerImage.cs deleted file mode 100644 index a252a26..0000000 --- a/CodexDistTestCore/Config/CodexDockerImage.cs +++ /dev/null @@ -1,72 +0,0 @@ -using k8s.Models; - -namespace CodexDistTestCore.Config -{ - public class CodexDockerImage - { - public string GetImageTag() - { - return "thatbenbierens/nim-codex:sha-b204837"; - } - - public string GetExpectedImageRevision() - { - return "b20483"; - } - - public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer container) - { - var formatter = new EnvFormatter(); - formatter.Create(node, container); - return formatter.Result; - } - - private class EnvFormatter - { - public List Result { get; } = new List(); - - public void Create(OfflineCodexNodes node, CodexNodeContainer container) - { - AddVar("API_PORT", container.ApiPort.ToString()); - AddVar("DATA_DIR", container.DataDir); - AddVar("DISC_PORT", container.DiscoveryPort.ToString()); - AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}"); - - if (node.BootstrapNode != null) - { - var debugInfo = node.BootstrapNode.GetDebugInfo(); - AddVar("BOOTSTRAP_SPR", debugInfo.spr); - } - if (node.LogLevel != null) - { - AddVar("LOG_LEVEL", node.LogLevel.ToString()!.ToUpperInvariant()); - } - if (node.StorageQuota != null) - { - AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!); - } - if (node.MetricsEnabled) - { - AddVar("METRICS_ADDR", "0.0.0.0"); - AddVar("METRICS_PORT", container.MetricsPort.ToString()); - } - if (node.MarketplaceConfig != null) - { - //ETH_PROVIDER - //ETH_ACCOUNT - //ETH_DEPLOYMENT - AddVar("ETH_ACCOUNT", container.GethCompanionNodeContainer!.Account); - } - } - - private void AddVar(string key, string value) - { - Result.Add(new V1EnvVar - { - Name = key, - Value = value - }); - } - } - } -} diff --git a/CodexDistTestCore/Config/FileManagerConfig.cs b/CodexDistTestCore/Config/FileManagerConfig.cs deleted file mode 100644 index f7befc2..0000000 --- a/CodexDistTestCore/Config/FileManagerConfig.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace CodexDistTestCore.Config -{ - public class FileManagerConfig - { - public const string Folder = "TestDataFiles"; - } -} diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs deleted file mode 100644 index a17e6fd..0000000 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ /dev/null @@ -1,40 +0,0 @@ -using k8s; - -namespace CodexDistTestCore.Config -{ - public class K8sCluster - { - public const string K8sNamespace = ""; - private const string KubeConfigFile = "C:\\kube\\config"; - private readonly Dictionary K8sNodeLocationMap = new Dictionary - { - { Location.BensLaptop, "worker01" }, - { Location.BensOldGamingMachine, "worker02" }, - }; - - private KubernetesClientConfiguration? config; - - public KubernetesClientConfiguration GetK8sClientConfig() - { - if (config != null) return config; - //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); - config = KubernetesClientConfiguration.BuildDefaultConfig(); - return config; - } - - public string GetIp() - { - var c = GetK8sClientConfig(); - - var host = c.Host.Replace("https://", ""); - - return host.Substring(0, host.IndexOf(':')); - } - - public string GetNodeLabelForLocation(Location location) - { - if (location == Location.Unspecified) return string.Empty; - return K8sNodeLocationMap[location]; - } - } -} diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs deleted file mode 100644 index bf95aa7..0000000 --- a/CodexDistTestCore/DistTest.cs +++ /dev/null @@ -1,120 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - [SetUpFixture] - public abstract class DistTest - { - private TestLog log = null!; - private FileManager fileManager = null!; - public K8sManager k8sManager = null!; - - [OneTimeSetUp] - public void GlobalSetup() - { - // Previous test run may have been interrupted. - // Begin by cleaning everything up. - log = new TestLog(); - fileManager = new FileManager(log); - k8sManager = new K8sManager(log, fileManager); - - try - { - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); - } - catch (Exception ex) - { - GlobalTestFailure.HasFailed = true; - log.Error($"Global setup cleanup failed with: {ex}"); - throw; - } - log.Log("Global setup cleanup successful"); - } - - [SetUp] - public void SetUpDistTest() - { - if (GlobalTestFailure.HasFailed) - { - Assert.Inconclusive("Skip test: Previous test failed during clean up."); - } - else - { - var dockerImage = new CodexDockerImage(); - log = new TestLog(); - log.Log($"Using docker image '{dockerImage.GetImageTag()}'"); - - fileManager = new FileManager(log); - k8sManager = new K8sManager(log, fileManager); - } - } - - [TearDown] - public void TearDownDistTest() - { - try - { - log.EndTest(); - IncludeLogsAndMetricsOnTestFailure(); - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); - } - catch (Exception ex) - { - log.Error("Cleanup failed: " + ex.Message); - GlobalTestFailure.HasFailed = true; - } - } - - public TestFile GenerateTestFile(ByteSize size) - { - return fileManager.GenerateTestFile(size); - } - - public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes) - { - return new OfflineCodexNodes(k8sManager, numberOfNodes); - } - - private void IncludeLogsAndMetricsOnTestFailure() - { - var result = TestContext.CurrentContext.Result; - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - if (IsDownloadingLogsAndMetricsEnabled()) - { - log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - k8sManager.ForEachOnlineGroup(DownloadLogs); - k8sManager.DownloadAllMetrics(); - } - else - { - log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - } - } - } - - private void DownloadLogs(CodexNodeGroup group) - { - foreach (var node in group) - { - var downloader = new PodLogDownloader(log, k8sManager); - var n = (OnlineCodexNode)node; - downloader.DownloadLog(n); - } - } - - private bool IsDownloadingLogsAndMetricsEnabled() - { - var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); - } - } - - public static class GlobalTestFailure - { - public static bool HasFailed { get; set; } = false; - } -} diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs deleted file mode 100644 index 6fbd55f..0000000 --- a/CodexDistTestCore/FileManager.cs +++ /dev/null @@ -1,110 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IFileManager - { - TestFile CreateEmptyTestFile(); - TestFile GenerateTestFile(ByteSize size); - void DeleteAllTestFiles(); - } - - public class FileManager : IFileManager - { - public const int ChunkSize = 1024 * 1024; - private readonly Random random = new Random(); - private readonly List activeFiles = new List(); - private readonly TestLog log; - - public FileManager(TestLog log) - { - if (!Directory.Exists(FileManagerConfig.Folder)) Directory.CreateDirectory(FileManagerConfig.Folder); - this.log = log; - } - - public TestFile CreateEmptyTestFile() - { - var result = new TestFile(Path.Combine(FileManagerConfig.Folder, Guid.NewGuid().ToString() + "_test.bin")); - File.Create(result.Filename).Close(); - activeFiles.Add(result); - return result; - } - - public TestFile GenerateTestFile(ByteSize size) - { - var result = CreateEmptyTestFile(); - GenerateFileBytes(result, size); - log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); - return result; - } - - public void DeleteAllTestFiles() - { - foreach (var file in activeFiles) File.Delete(file.Filename); - activeFiles.Clear(); - } - - private void GenerateFileBytes(TestFile result, ByteSize size) - { - long bytesLeft = size.SizeInBytes; - while (bytesLeft > 0) - { - var length = Math.Min(bytesLeft, ChunkSize); - AppendRandomBytesToFile(result, length); - bytesLeft -= length; - } - } - - private void AppendRandomBytesToFile(TestFile result, long length) - { - var bytes = new byte[length]; - random.NextBytes(bytes); - using var stream = new FileStream(result.Filename, FileMode.Append); - stream.Write(bytes, 0, bytes.Length); - } - } - - public class TestFile - { - public TestFile(string filename) - { - Filename = filename; - } - - public string Filename { get; } - - public long GetFileSize() - { - var info = new FileInfo(Filename); - return info.Length; - } - - public void AssertIsEqual(TestFile? actual) - { - if (actual == null) Assert.Fail("TestFile is null."); - if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); - - Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); - - using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); - using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); - - var bytesExpected = new byte[FileManager.ChunkSize]; - var bytesActual = new byte[FileManager.ChunkSize]; - - var readExpected = 0; - var readActual = 0; - - while (true) - { - readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); - readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); - - if (readExpected == 0 && readActual == 0) return; - Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); - CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); - } - } - } -} diff --git a/CodexDistTestCore/Http.cs b/CodexDistTestCore/Http.cs deleted file mode 100644 index fd7e31a..0000000 --- a/CodexDistTestCore/Http.cs +++ /dev/null @@ -1,100 +0,0 @@ -using Newtonsoft.Json; -using NUnit.Framework; -using System.Net.Http.Headers; - -namespace CodexDistTestCore -{ - public class Http - { - private readonly string ip; - private readonly int port; - private readonly string baseUrl; - - public Http(string ip, int port, string baseUrl) - { - this.ip = ip; - this.port = port; - this.baseUrl = baseUrl; - - if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; - if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; - } - - public string HttpGetString(string route) - { - return Retry(() => - { - using var client = GetClient(); - var url = GetUrl() + route; - var result = Utils.Wait(client.GetAsync(url)); - return Utils.Wait(result.Content.ReadAsStringAsync()); - }); - } - - public T HttpGetJson(string route) - { - return JsonConvert.DeserializeObject(HttpGetString(route))!; - } - - public string HttpPostStream(string route, Stream stream) - { - return Retry(() => - { - using var client = GetClient(); - var url = GetUrl() + route; - - var content = new StreamContent(stream); - content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - var response = Utils.Wait(client.PostAsync(url, content)); - - return Utils.Wait(response.Content.ReadAsStringAsync()); - }); - } - - public Stream HttpGetStream(string route) - { - return Retry(() => - { - var client = GetClient(); - var url = GetUrl() + route; - - return Utils.Wait(client.GetStreamAsync(url)); - }); - } - - private string GetUrl() - { - return $"http://{ip}:{port}{baseUrl}"; - } - - private static T Retry(Func operation) - { - var retryCounter = 0; - - while (true) - { - try - { - return operation(); - } - catch (Exception exception) - { - Timing.HttpCallRetryDelay(); - retryCounter++; - if (retryCounter > Timing.HttpCallRetryCount()) - { - Assert.Fail(exception.Message); - throw; - } - } - } - } - - private static HttpClient GetClient() - { - var client = new HttpClient(); - client.Timeout = Timing.HttpCallTimeout(); - return client; - } - } -} diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs deleted file mode 100644 index 06bcafb..0000000 --- a/CodexDistTestCore/K8sManager.cs +++ /dev/null @@ -1,177 +0,0 @@ -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; - -namespace CodexDistTestCore -{ - public interface IK8sManager - { - ICodexNodeGroup BringOnline(OfflineCodexNodes node); - IOfflineCodexNodes BringOffline(ICodexNodeGroup node); - void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler); - } - - public class K8sManager : IK8sManager - { - private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource(); - private readonly List onlineCodexNodeGroups = new List(); - private readonly KnownK8sPods knownPods = new KnownK8sPods(); - private readonly TestLog log; - private readonly IFileManager fileManager; - private readonly MetricsAggregator metricsAggregator; - private readonly MarketplaceController marketplaceController; - - public K8sManager(TestLog log, IFileManager fileManager) - { - this.log = log; - this.fileManager = fileManager; - metricsAggregator = new MetricsAggregator(log, this); - marketplaceController = new MarketplaceController(log, this); - } - - public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) - { - var group = CreateOnlineCodexNodes(offline); - - if (offline.MarketplaceConfig != null) - { - group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline); - ConnectMarketplace(group); - } - - K8s(k => k.BringOnline(group, offline)); - - if (offline.MetricsEnabled) - { - BringOnlineMetrics(group); - } - - log.Log($"{group.Describe()} online."); - - return group; - } - - public IOfflineCodexNodes BringOffline(ICodexNodeGroup node) - { - var online = GetAndRemoveActiveNodeFor(node); - - K8s(k => k.BringOffline(online)); - - log.Log($"{online.Describe()} offline."); - - return online.Origin; - } - - public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) - { - return K8s(k => k.ExecuteCommand(pod, containerName, command, arguments)); - } - - public void DeleteAllResources() - { - K8s(k => k.DeleteAllResources()); - } - - public void ForEachOnlineGroup(Action action) - { - foreach (var group in onlineCodexNodeGroups) action(group); - } - - public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler) - { - K8s(k => k.FetchPodLog(node, logHandler)); - } - - public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber) - { - var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config); - - return K8s(k => k.BringOnlinePrometheus(spec)); - } - - public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec() - { - return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); - } - - public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) - { - return K8s(k => k.BringOnlineGethBootstrapNode(spec)); - } - - public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) - { - return K8s(k => k.BringOnlineGethCompanionGroup(info, group)); - } - - public void DownloadAllMetrics() - { - metricsAggregator.DownloadAllMetrics(); - } - - private void BringOnlineMetrics(CodexNodeGroup group) - { - metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group)); - } - - private void ConnectMarketplace(CodexNodeGroup group) - { - for (var i = 0; i < group.Nodes.Length; i++) - { - ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]); - } - } - - private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container) - { - node.Container.GethCompanionNodeContainer = container; // :c - - var access = new MarketplaceAccess(this, marketplaceController, log, group, container); - access.Initialize(); - node.Marketplace = access; - } - - private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) - { - var containers = CreateContainers(offline); - var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); - var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); - onlineCodexNodeGroups.Add(result); - return result; - } - - private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline) - { - var factory = new CodexNodeContainerFactory(codexGroupNumberSource); - var containers = new List(); - for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline)); - return containers.ToArray(); - } - - private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node) - { - var n = (CodexNodeGroup)node; - onlineCodexNodeGroups.Remove(n); - return n; - } - - private void K8s(Action action) - { - var k8s = new K8sOperations(knownPods); - action(k8s); - k8s.Close(); - } - - private T K8s(Func action) - { - var k8s = new K8sOperations(knownPods); - var result = action(k8s); - k8s.Close(); - return result; - } - - private static OnlineCodexNode[] DowncastNodes(CodexNodeGroup group) - { - return group.Nodes.Cast().ToArray(); - } - } -} diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs deleted file mode 100644 index 0cc3ea1..0000000 --- a/CodexDistTestCore/K8sOperations.cs +++ /dev/null @@ -1,349 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; -using k8s; -using k8s.Models; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public class K8sOperations - { - private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly Kubernetes client; - private readonly KnownK8sPods knownPods; - - public K8sOperations(KnownK8sPods knownPods) - { - this.knownPods = knownPods; - - client = new Kubernetes(k8sCluster.GetK8sClientConfig()); - } - - public void Close() - { - client.Dispose(); - } - - public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) - { - EnsureTestNamespace(); - - CreateDeployment(online, offline); - CreateService(online); - - WaitUntilOnline(online); - FetchPodInfo(online); - } - - public void BringOffline(CodexNodeGroup online) - { - var deploymentName = online.Deployment.Name(); - DeleteDeployment(online); - DeleteService(online); - WaitUntilOffline(deploymentName); - } - - public void DeleteAllResources() - { - DeleteNamespace(); - - WaitUntilZeroPods(); - WaitUntilNamespaceDeleted(); - } - - public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler) - { - var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name); - logHandler.Log(stream); - } - - public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) - { - var runner = new CommandRunner(client, pod, containerName, command, arguments); - runner.Run(); - return runner.GetStdOut(); - } - - public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec) - { - EnsureTestNamespace(); - - CreatePrometheusDeployment(spec); - CreatePrometheusService(spec); - WaitUntilPrometheusOnline(spec); - - return new PrometheusInfo(spec.ServicePort, FetchNewPod()); - } - - public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) - { - EnsureTestNamespace(); - - CreateGethBootstrapDeployment(spec); - CreateGethBootstrapService(spec); - WaitUntilGethBootstrapOnline(spec); - - return FetchNewPod(); - } - - public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) - { - EnsureTestNamespace(); - - CreateGethCompanionDeployment(info, group); - WaitUntilGethCompanionGroupOnline(info.Spec, group); - - return FetchNewPod(); - } - - private void FetchPodInfo(CodexNodeGroup online) - { - online.PodInfo = FetchNewPod(); - } - - private PodInfo FetchNewPod() - { - var pods = client.ListNamespacedPod(K8sNamespace).Items; - - var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); - Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); - - var newPod = newPods.Single(); - var info = new PodInfo(newPod.Name(), newPod.Status.PodIP); - - Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure."); - Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure."); - - knownPods.Add(newPod.Name()); - return info; - } - - #region Waiting - - private void WaitUntilOnline(CodexNodeGroup online) - { - WaitUntil(() => - { - online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; - }); - } - - private void WaitUntilOffline(string deploymentName) - { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment == null || deployment.Status.AvailableReplicas == 0; - }); - } - - private void WaitUntilZeroPods() - { - WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); - } - - private void WaitUntilNamespaceDeleted() - { - WaitUntil(() => !IsTestNamespaceOnline()); - } - - private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) - { - WaitUntilDeploymentOnline(spec.GetDeploymentName()); - } - - private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) - { - WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); - } - - private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) - { - WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); - } - - private void WaitUntilDeploymentOnline(string deploymentName) - { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; - }); - } - - private void WaitUntil(Func predicate) - { - var start = DateTime.UtcNow; - var state = predicate(); - while (!state) - { - if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) - { - Assert.Fail("K8s operation timed out."); - throw new TimeoutException(); - } - - Timing.WaitForK8sServiceDelay(); - state = predicate(); - } - } - - #endregion - - #region Service management - - private void CreateService(CodexNodeGroup online) - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = online.GetServiceMetadata(), - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = online.GetSelector(), - Ports = CreateServicePorts(online) - } - }; - - online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); - } - - private List CreateServicePorts(CodexNodeGroup online) - { - var result = new List(); - var containers = online.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1ServicePort - { - Name = container.ServicePortName, - Protocol = "TCP", - Port = container.ApiPort, - TargetPort = container.ContainerPortName, - NodePort = container.ServicePort - }); - } - return result; - } - - private void DeleteService(CodexNodeGroup online) - { - if (online.Service == null) return; - client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); - online.Service = null; - } - - private void CreatePrometheusService(K8sPrometheusSpecs spec) - { - client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); - } - - private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec) - { - client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace); - } - - #endregion - - #region Deployment management - - private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = online.GetDeploymentMetadata(), - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = online.GetSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = online.GetSelector() - }, - Spec = new V1PodSpec - { - NodeSelector = CreateNodeSelector(offline), - Containers = CreateDeploymentContainers(online, offline) - } - } - } - }; - - online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - } - - private IDictionary CreateNodeSelector(OfflineCodexNodes offline) - { - if (offline.Location == Location.Unspecified) return new Dictionary(); - - return new Dictionary - { - { "codex-test-location", k8sCluster.GetNodeLabelForLocation(offline.Location) } - }; - } - - private List CreateDeploymentContainers(CodexNodeGroup group, OfflineCodexNodes offline) - { - var result = new List(); - var containers = group.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1Container - { - Name = container.Name, - Image = dockerImage.GetImageTag(), - Ports = new List - { - new V1ContainerPort - { - ContainerPort = container.ApiPort, - Name = container.ContainerPortName - } - }, - Env = dockerImage.CreateEnvironmentVariables(offline, container) - }); - } - - return result; - } - - private void DeleteDeployment(CodexNodeGroup group) - { - if (group.Deployment == null) return; - client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace); - group.Deployment = null; - } - - private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) - { - client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); - } - - private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec) - { - client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); - } - - private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) - { - client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); - } - - #endregion - - private class CommandRunner - { - - } - } -} diff --git a/CodexDistTestCore/KnownK8sPods.cs b/CodexDistTestCore/KnownK8sPods.cs deleted file mode 100644 index 940a147..0000000 --- a/CodexDistTestCore/KnownK8sPods.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace CodexDistTestCore -{ - public class KnownK8sPods - { - private readonly List knownActivePodNames = new List(); - - public bool Contains(string name) - { - return knownActivePodNames.Contains(name); - } - - public void Add(string name) - { - knownActivePodNames.Add(name); - } - } -} diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs deleted file mode 100644 index 74d687c..0000000 --- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace CodexDistTestCore.Marketplace -{ - public class GethCompanionGroup - { - public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers) - { - Number = number; - Containers = containers; - } - - public int Number { get; } - public GethCompanionNodeContainer[] Containers { get; } - public PodInfo? Pod { get; set; } - } - - public class GethCompanionNodeContainer - { - public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName, int authRpcPort) - { - Name = name; - ApiPort = apiPort; - AuthRpcPort = authRpcPort; - RpcPort = rpcPort; - ContainerPortName = containerPortName; - } - - public string Name { get; } - public int ApiPort { get; } - public int AuthRpcPort { get; } - public int RpcPort { get; } - public string ContainerPortName { get; } - - public string Account { get; set; } = string.Empty; - } -} diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs deleted file mode 100644 index bfaf3d4..0000000 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ /dev/null @@ -1,207 +0,0 @@ -using CodexDistTestCore.Config; -using k8s.Models; - -namespace CodexDistTestCore.Marketplace -{ - public static class GethDockerImage - { - public const string Image = "thatbenbierens/geth-confenv:latest"; - - } - - public class K8sGethBoostrapSpecs - { - public const string ContainerName = "dtest-gethb"; - private const string portName = "gethb"; - - public K8sGethBoostrapSpecs(int servicePort) - { - ServicePort = servicePort; - } - - public int ServicePort { get; } - - public string GetBootstrapDeploymentName() - { - return "test-gethb"; - } - - public string GetCompanionDeploymentName(GethCompanionGroup group) - { - return "test-geth" + group.Number; - } - - public V1Deployment CreateGethBootstrapDeployment() - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetBootstrapDeploymentName(), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateBootstrapSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateBootstrapSelector() - }, - Spec = new V1PodSpec - { - Containers = new List - { - new V1Container - { - Name = ContainerName, - Image = GethDockerImage.Image, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 8545, - Name = portName - } - }, - Env = new List - { - new V1EnvVar - { - Name = "GETH_ARGS", - Value = "" - }, - new V1EnvVar - { - Name = "GENESIS_JSON", - Value = genesisJsonBase64 - }, - new V1EnvVar - { - Name = "IS_BOOTSTRAP", - Value = "1" - } - } - } - } - } - } - } - }; - - return deploymentSpec; - } - - public V1Service CreateGethBootstrapService() - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = "codex-gethb-service", - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = CreateBootstrapSelector(), - Ports = new List - { - new V1ServicePort - { - Name = "gethb-service", - Protocol = "TCP", - Port = 8545, - TargetPort = portName, - NodePort = ServicePort - } - } - } - }; - - return serviceSpec; - } - - public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetCompanionDeploymentName(group), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateCompanionSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateCompanionSelector() - }, - Spec = new V1PodSpec - { - Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList() - } - } - } - }; - - return deploymentSpec; - } - - private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info) - { - return new V1Container - { - Name = container.Name, - Image = GethDockerImage.Image, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = container.ApiPort, - Name = container.ContainerPortName - } - }, - // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 - Env = new List - { - new V1EnvVar - { - Name = "GETH_ARGS", - Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.AuthRpcPort} --http.port {container.RpcPort}" - }, - new V1EnvVar - { - Name = "GENESIS_JSON", - Value = info.GenesisJsonBase64 - } - } - }; - } - - private Dictionary CreateBootstrapSelector() - { - return new Dictionary { { "test-gethb", "dtest-gethb" } }; - } - - private Dictionary CreateCompanionSelector() - { - return new Dictionary { { "test-gethc", "dtest-gethc" } }; - } - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs deleted file mode 100644 index e9bf39e..0000000 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ /dev/null @@ -1,111 +0,0 @@ -using NUnit.Framework; -using NUnit.Framework.Constraints; - -namespace CodexDistTestCore.Marketplace -{ - public interface IMarketplaceAccess - { - void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral); - void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes); - void AssertThatBalance(IResolveConstraint constraint, string message = ""); - decimal GetBalance(); - } - - public class MarketplaceAccess : IMarketplaceAccess - { - private readonly K8sManager k8sManager; - private readonly MarketplaceController marketplaceController; - private readonly TestLog log; - private readonly CodexNodeGroup group; - private readonly GethCompanionNodeContainer container; - - public MarketplaceAccess( - K8sManager k8sManager, - MarketplaceController marketplaceController, - TestLog log, - CodexNodeGroup group, - GethCompanionNodeContainer container) - { - this.k8sManager = k8sManager; - this.marketplaceController = marketplaceController; - this.log = log; - this.group = group; - this.container = container; - } - - public void Initialize() - { - EnsureAccount(); - - marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); - - log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); - } - - public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) - { - throw new NotImplementedException(); - } - - public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) - { - throw new NotImplementedException(); - } - - public void AssertThatBalance(IResolveConstraint constraint, string message = "") - { - throw new NotImplementedException(); - } - - public decimal GetBalance() - { - return marketplaceController.GetBalance(container.Account); - } - - private void EnsureAccount() - { - FetchAccount(); - if (string.IsNullOrEmpty(container.Account)) - { - Thread.Sleep(TimeSpan.FromSeconds(15)); - FetchAccount(); - } - Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); - } - - private void FetchAccount() - { - container.Account = k8sManager.ExecuteCommand(group.GethCompanionGroup!.Pod!, container.Name, "cat", GethDockerImage.AccountFilename); - } - } - - public class MarketplaceUnavailable : IMarketplaceAccess - { - public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) - { - Unavailable(); - } - - public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) - { - Unavailable(); - } - - public void AssertThatBalance(IResolveConstraint constraint, string message = "") - { - Unavailable(); - } - - public decimal GetBalance() - { - Unavailable(); - return 0; - } - - private void Unavailable() - { - Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it."); - throw new InvalidOperationException(); - } - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs deleted file mode 100644 index 8bab9e9..0000000 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ /dev/null @@ -1,24 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; -using System.Numerics; -using System.Text; - -namespace CodexDistTestCore.Marketplace -{ - public class MarketplaceController - { - private readonly TestLog log; - private readonly K8sManager k8sManager; - private readonly NumberSource companionGroupNumberSource = new NumberSource(0); - private List companionGroups = new List(); - private GethBootstrapInfo? bootstrapInfo; - - public MarketplaceController(TestLog log, K8sManager k8sManager) - { - this.log = log; - this.k8sManager = k8sManager; - } - - - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs deleted file mode 100644 index 23ad25f..0000000 --- a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ /dev/null @@ -1,4 +0,0 @@ -namespace CodexDistTestCore.Marketplace -{ - -} diff --git a/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs b/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs deleted file mode 100644 index 0fc0865..0000000 --- a/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs +++ /dev/null @@ -1,122 +0,0 @@ -using CodexDistTestCore.Config; -using k8s.Models; - -namespace CodexDistTestCore.Metrics -{ - public class K8sPrometheusSpecs - { - public const string ContainerName = "dtest-prom"; - public const string ConfigFilepath = "/etc/prometheus/prometheus.yml"; - private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; - private const string portName = "prom-1"; - private readonly string config; - - public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config) - { - ServicePort = servicePort; - PrometheusNumber = prometheusNumber; - this.config = config; - } - - public int ServicePort { get; } - public int PrometheusNumber { get; } - - public string GetDeploymentName() - { - return "test-prom" + PrometheusNumber; - } - - public V1Deployment CreatePrometheusDeployment() - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetDeploymentName(), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateSelector() - }, - Spec = new V1PodSpec - { - Containers = new List - { - new V1Container - { - Name = ContainerName, - Image = dockerImage, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 9090, - Name = portName - } - }, - Env = new List - { - new V1EnvVar - { - Name = "PROM_CONFIG", - Value = config - } - } - } - } - } - } - } - }; - - return deploymentSpec; - } - - public V1Service CreatePrometheusService() - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = "codex-prom-service" + PrometheusNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = CreateSelector(), - Ports = new List - { - new V1ServicePort - { - Name = "prom-service" + PrometheusNumber, - Protocol = "TCP", - Port = 9090, - TargetPort = portName, - NodePort = ServicePort - } - } - } - }; - - return serviceSpec; - } - - private Dictionary CreateSelector() - { - return new Dictionary { { "test-prom", "dtest-prom" } }; - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsAccess.cs b/CodexDistTestCore/Metrics/MetricsAccess.cs deleted file mode 100644 index 2f6456e..0000000 --- a/CodexDistTestCore/Metrics/MetricsAccess.cs +++ /dev/null @@ -1,63 +0,0 @@ -using NUnit.Framework; -using NUnit.Framework.Constraints; - -namespace CodexDistTestCore.Metrics -{ - public interface IMetricsAccess - { - void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); - } - - public class MetricsUnavailable : IMetricsAccess - { - public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") - { - Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); - throw new InvalidOperationException(); - } - } - - public class MetricsAccess : IMetricsAccess - { - private readonly MetricsQuery query; - private readonly OnlineCodexNode node; - - public MetricsAccess(MetricsQuery query, OnlineCodexNode node) - { - this.query = query; - this.node = node; - } - - public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") - { - var metricSet = GetMetricWithTimeout(metricName, node); - var metricValue = metricSet.Values[0].Value; - Assert.That(metricValue, constraint, message); - } - - private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node) - { - var start = DateTime.UtcNow; - - while (true) - { - var mostRecent = GetMostRecent(metricName, node); - if (mostRecent != null) return mostRecent; - if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) - { - Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); - throw new TimeoutException(); - } - - Utils.Sleep(TimeSpan.FromSeconds(2)); - } - } - - private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node) - { - var result = query.GetMostRecent(metricName, node); - if (result == null) return null; - return result.Sets.LastOrDefault(); - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsAggregator.cs b/CodexDistTestCore/Metrics/MetricsAggregator.cs deleted file mode 100644 index adf0c3f..0000000 --- a/CodexDistTestCore/Metrics/MetricsAggregator.cs +++ /dev/null @@ -1,78 +0,0 @@ -using NUnit.Framework; -using System.Text; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsAggregator - { - private readonly NumberSource prometheusNumberSource = new NumberSource(0); - private readonly TestLog log; - private readonly K8sManager k8sManager; - private readonly Dictionary activePrometheuses = new Dictionary(); - - public MetricsAggregator(TestLog log, K8sManager k8sManager) - { - this.log = log; - this.k8sManager = k8sManager; - } - - public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes) - { - log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); - - var config = GeneratePrometheusConfig(nodes); - var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber()); - var query = new MetricsQuery(prometheus); - activePrometheuses.Add(query, nodes); - - log.Log("Metrics service started."); - - foreach (var node in nodes) - { - node.Metrics = new MetricsAccess(query, node); - } - } - - public void DownloadAllMetrics() - { - var download = new MetricsDownloader(log, activePrometheuses); - download.DownloadAllMetrics(); - } - - private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) - { - var config = ""; - config += "global:\n"; - config += " scrape_interval: 30s\n"; - config += " scrape_timeout: 10s\n"; - config += "\n"; - config += "scrape_configs:\n"; - config += " - job_name: services\n"; - config += " metrics_path: /metrics\n"; - config += " static_configs:\n"; - config += " - targets:\n"; - - foreach (var node in nodes) - { - var ip = node.Group.PodInfo!.Ip; - var port = node.Container.MetricsPort; - config += $" - '{ip}:{port}'\n"; - } - - var bytes = Encoding.ASCII.GetBytes(config); - return Convert.ToBase64String(bytes); - } - } - - public class PrometheusInfo - { - public PrometheusInfo(int servicePort, PodInfo podInfo) - { - ServicePort = servicePort; - PodInfo = podInfo; - } - - public int ServicePort { get; } - public PodInfo PodInfo { get; } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsDownloader.cs b/CodexDistTestCore/Metrics/MetricsDownloader.cs deleted file mode 100644 index 18fd10b..0000000 --- a/CodexDistTestCore/Metrics/MetricsDownloader.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System.Globalization; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsDownloader - { - private readonly TestLog log; - private readonly Dictionary activePrometheuses; - - public MetricsDownloader(TestLog log, Dictionary activePrometheuses) - { - this.log = log; - this.activePrometheuses = activePrometheuses; - } - - public void DownloadAllMetrics() - { - foreach (var pair in activePrometheuses) - { - DownloadAllMetrics(pair.Key, pair.Value); - } - } - - private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) - { - foreach (var node in nodes) - { - DownloadAllMetricsForNode(query, node); - } - } - - private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) - { - var metrics = query.GetAllMetricsForNode(node); - if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; - - var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); - var map = CreateValueMap(metrics); - - WriteToFile(node.GetName(), headers, map); - } - - private void WriteToFile(string nodeName, string[] headers, Dictionary> map) - { - var file = log.CreateSubfile("csv"); - log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); - - file.WriteRaw(string.Join(",", headers)); - - foreach (var pair in map) - { - file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); - } - } - - private Dictionary> CreateValueMap(Metrics metrics) - { - var map = CreateForAllTimestamps(metrics); - foreach (var metric in metrics.Sets) - { - AddToMap(map, metric); - } - return map; - - } - - private Dictionary> CreateForAllTimestamps(Metrics metrics) - { - var result = new Dictionary>(); - var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); - foreach (var timestamp in timestamps) result.Add(timestamp, new List()); - return result; - } - - private void AddToMap(Dictionary> map, MetricsSet metric) - { - foreach (var key in map.Keys) - { - map[key].Add(GetValueAtTimestamp(key, metric)); - } - } - - private string GetValueAtTimestamp(DateTime key, MetricsSet metric) - { - var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); - if (value == null) return ""; - return value.Value.ToString(CultureInfo.InvariantCulture); - } - - private string FormatTimestamp(DateTime key) - { - var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - var diff = key - origin; - return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsQuery.cs b/CodexDistTestCore/Metrics/MetricsQuery.cs deleted file mode 100644 index c028a6c..0000000 --- a/CodexDistTestCore/Metrics/MetricsQuery.cs +++ /dev/null @@ -1,190 +0,0 @@ -using CodexDistTestCore.Config; -using System.Globalization; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsQuery - { - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly Http http; - - public MetricsQuery(PrometheusInfo prometheusInfo) - { - http = new Http( - k8sCluster.GetIp(), - prometheusInfo.ServicePort, - "api/v1"); - } - - public Metrics? GetMostRecent(string metricName, OnlineCodexNode node) - { - var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); - if (response == null) return null; - - return new Metrics - { - Sets = response.data.result.Select(r => - { - return new MetricsSet - { - Instance = r.metric.instance, - Values = MapSingleValue(r.value) - }; - }).ToArray() - }; - } - - public Metrics? GetMetrics(string metricName) - { - var response = GetAll(metricName); - if (response == null) return null; - return MapResponseToMetrics(response); - } - - public Metrics? GetAllMetricsForNode(OnlineCodexNode node) - { - var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); - if (response.status != "success") return null; - return MapResponseToMetrics(response); - } - - private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) - { - var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); - if (response.status != "success") return null; - return response; - } - - private PrometheusQueryResponse? GetAll(string metricName) - { - var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); - if (response.status != "success") return null; - return response; - } - - private Metrics MapResponseToMetrics(PrometheusQueryResponse response) - { - return new Metrics - { - Sets = response.data.result.Select(r => - { - return new MetricsSet - { - Name = r.metric.__name__, - Instance = r.metric.instance, - Values = MapMultipleValues(r.values) - }; - }).ToArray() - }; - } - - private MetricsSetValue[] MapSingleValue(object[] value) - { - if (value != null && value.Length > 0) - { - return new[] - { - MapValue(value) - }; - } - return Array.Empty(); - } - - private MetricsSetValue[] MapMultipleValues(object[][] values) - { - if (values != null && values.Length > 0) - { - return values.Select(v => MapValue(v)).ToArray(); - } - return Array.Empty(); - } - - private MetricsSetValue MapValue(object[] value) - { - if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); - - return new MetricsSetValue - { - Timestamp = ToTimestamp(value[0]), - Value = ToValue(value[1]) - }; - } - - private string GetInstanceNameForNode(OnlineCodexNode node) - { - var pod = node.Group.PodInfo!; - return $"{pod.Ip}:{node.Container.MetricsPort}"; - } - - private string GetInstanceStringForNode(OnlineCodexNode node) - { - return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; - } - - private string GetQueryTimeRange() - { - return "[12h]"; - } - - private double ToValue(object v) - { - return Convert.ToDouble(v, CultureInfo.InvariantCulture); - } - - private DateTime ToTimestamp(object v) - { - var unixSeconds = ToValue(v); - return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); - } - } - - public class Metrics - { - public MetricsSet[] Sets { get; set; } = Array.Empty(); - } - - public class MetricsSet - { - public string Name { get; set; } = string.Empty; - public string Instance { get; set; } = string.Empty; - public MetricsSetValue[] Values { get; set; } = Array.Empty(); - } - - public class MetricsSetValue - { - public DateTime Timestamp { get; set; } - public double Value { get; set; } - } - - public class PrometheusQueryResponse - { - public string status { get; set; } = string.Empty; - public PrometheusQueryResponseData data { get; set; } = new(); - } - - public class PrometheusQueryResponseData - { - public string resultType { get; set; } = string.Empty; - public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); - } - - public class PrometheusQueryResponseDataResultEntry - { - public ResultEntryMetric metric { get; set; } = new(); - public object[] value { get; set; } = Array.Empty(); - public object[][] values { get; set; } = Array.Empty(); - } - - public class ResultEntryMetric - { - public string __name__ { get; set; } = string.Empty; - public string instance { get; set; } = string.Empty; - public string job { get; set; } = string.Empty; - } - - public class PrometheusAllNamesResponse - { - public string status { get; set; } = string.Empty; - public string[] data { get; set; } = Array.Empty(); - } -} diff --git a/CodexDistTestCore/NumberSource.cs b/CodexDistTestCore/NumberSource.cs deleted file mode 100644 index 4338610..0000000 --- a/CodexDistTestCore/NumberSource.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace CodexDistTestCore -{ - public class NumberSource - { - private int number; - - public NumberSource(int start) - { - number = start; - } - - public int GetNextNumber() - { - var n = number; - number++; - return n; - } - } -} diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs deleted file mode 100644 index 84003a6..0000000 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ /dev/null @@ -1,97 +0,0 @@ -using CodexDistTestCore.Marketplace; - -namespace CodexDistTestCore -{ - public interface IOfflineCodexNodes - { - IOfflineCodexNodes At(Location location); - IOfflineCodexNodes WithLogLevel(CodexLogLevel level); - IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); - IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); - IOfflineCodexNodes EnableMetrics(); - IOfflineCodexNodes EnableMarketplace(int initialBalance); - ICodexNodeGroup BringOnline(); - } - - public enum Location - { - Unspecified, - BensLaptop, - BensOldGamingMachine, - } - - public class OfflineCodexNodes : IOfflineCodexNodes - { - private readonly IK8sManager k8SManager; - - public int NumberOfNodes { get; } - public Location Location { get; private set; } - public CodexLogLevel? LogLevel { get; private set; } - public IOnlineCodexNode? BootstrapNode { get; private set; } - public ByteSize? StorageQuota { get; private set; } - public bool MetricsEnabled { get; private set; } - public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } - - public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) - { - this.k8SManager = k8SManager; - NumberOfNodes = numberOfNodes; - Location = Location.Unspecified; - MetricsEnabled = false; - } - - public ICodexNodeGroup BringOnline() - { - return k8SManager.BringOnline(this); - } - - public IOfflineCodexNodes At(Location location) - { - Location = location; - return this; - } - - public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node) - { - BootstrapNode = node; - return this; - } - - public IOfflineCodexNodes WithLogLevel(CodexLogLevel level) - { - LogLevel = level; - return this; - } - - public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota) - { - StorageQuota = storageQuota; - return this; - } - - public IOfflineCodexNodes EnableMetrics() - { - MetricsEnabled = true; - return this; - } - - public IOfflineCodexNodes EnableMarketplace(int initialBalance) - { - MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); - return this; - } - - public string Describe() - { - var args = string.Join(',', DescribeArgs()); - return $"{NumberOfNodes} CodexNodes with [{args}]"; - } - - private IEnumerable DescribeArgs() - { - if (LogLevel != null) yield return ($"LogLevel={LogLevel}"); - if (BootstrapNode != null) yield return ("BootstrapNode=set-not-shown-here"); - if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}"); - } - } -} diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs deleted file mode 100644 index 64f9c6d..0000000 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ /dev/null @@ -1,141 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IOnlineCodexNode - { - CodexDebugResponse GetDebugInfo(); - ContentId UploadFile(TestFile file); - TestFile? DownloadContent(ContentId contentId); - void ConnectToPeer(IOnlineCodexNode node); - ICodexNodeLog DownloadLog(); - IMetricsAccess Metrics { get; } - IMarketplaceAccess Marketplace { get; } - } - - public class OnlineCodexNode : IOnlineCodexNode - { - private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; - private const string UploadFailedMessage = "Unable to store block"; - - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly TestLog log; - private readonly IFileManager fileManager; - - public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container) - { - this.log = log; - this.fileManager = fileManager; - Container = container; - } - - public CodexNodeContainer Container { get; } - public CodexNodeGroup Group { get; internal set; } = null!; - public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable(); - public IMarketplaceAccess Marketplace { set; get; } = new MarketplaceUnavailable(); - - public string GetName() - { - return $"<{Container.Name}>"; - } - - public CodexDebugResponse GetDebugInfo() - { - var response = Http().HttpGetJson("debug/info"); - Log($"Got DebugInfo with id: '{response.id}'."); - return response; - } - - public ContentId UploadFile(TestFile file) - { - Log($"Uploading file of size {file.GetFileSize()}..."); - using var fileStream = File.OpenRead(file.Filename); - var response = Http().HttpPostStream("upload", fileStream); - if (response.StartsWith(UploadFailedMessage)) - { - Assert.Fail("Node failed to store block."); - } - Log($"Uploaded file. Received contentId: '{response}'."); - return new ContentId(response); - } - - public TestFile? DownloadContent(ContentId contentId) - { - Log($"Downloading for contentId: '{contentId.Id}'..."); - var file = fileManager.CreateEmptyTestFile(); - DownloadToFile(contentId.Id, file); - Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'."); - return file; - } - - public void ConnectToPeer(IOnlineCodexNode node) - { - var peer = (OnlineCodexNode)node; - - Log($"Connecting to peer {peer.GetName()}..."); - var peerInfo = node.GetDebugInfo(); - var peerId = peerInfo.id; - var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo); - - var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); - - Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); - Log($"Successfully connected to peer {peer.GetName()}."); - } - - public ICodexNodeLog DownloadLog() - { - return Group.DownloadLog(this); - } - - public string Describe() - { - return $"{Group.Describe()} contains {GetName()}"; - } - - private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) - { - var multiAddress = peerInfo.addrs.First(); - // Todo: Is there a case where First address in list is not the way? - - if (Group == peer.Group) - { - return multiAddress; - } - - // The peer we want to connect is in a different pod. - // We must replace the default IP with the pod IP in the multiAddress. - return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip); - } - - private void DownloadToFile(string contentId, TestFile file) - { - using var fileStream = File.OpenWrite(file.Filename); - using var downloadStream = Http().HttpGetStream("download/" + contentId); - downloadStream.CopyTo(fileStream); - } - - private Http Http() - { - return new Http(ip: k8sCluster.GetIp(), port: Container.ServicePort, baseUrl: "/api/codex/v1"); - } - - private void Log(string msg) - { - log.Log($"{GetName()}: {msg}"); - } - } - - public class ContentId - { - public ContentId(string id) - { - Id = id; - } - - public string Id { get; } - } -} diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs deleted file mode 100644 index e09a57c..0000000 --- a/CodexDistTestCore/PodLogDownloader.cs +++ /dev/null @@ -1,64 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IPodLogHandler - { - void Log(Stream log); - } - - public class PodLogDownloader - { - public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure"; - - private readonly TestLog log; - private readonly IK8sManager k8SManager; - - public PodLogDownloader(TestLog log, IK8sManager k8sManager) - { - this.log = log; - k8SManager = k8sManager; - } - - public CodexNodeLog DownloadLog(OnlineCodexNode node) - { - var description = node.Describe(); - var subFile = log.CreateSubfile(); - - log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); - var handler = new PodLogDownloadHandler(description, subFile); - k8SManager.FetchPodLog(node, handler); - return handler.CreateCodexNodeLog(); - } - } - - public class PodLogDownloadHandler : IPodLogHandler - { - private readonly string description; - private readonly LogFile log; - - public PodLogDownloadHandler(string description, LogFile log) - { - this.description = description; - this.log = log; - } - - public CodexNodeLog CreateCodexNodeLog() - { - return new CodexNodeLog(log); - } - - public void Log(Stream stream) - { - log.Write($"{description} -->> {log.FilenameWithoutPath}"); - log.WriteRaw(description); - var reader = new StreamReader(stream); - var line = reader.ReadLine(); - while (line != null) - { - log.WriteRaw(line); - line = reader.ReadLine(); - } - } - } -} diff --git a/CodexDistTestCore/Timing.cs b/CodexDistTestCore/Timing.cs deleted file mode 100644 index cfa9456..0000000 --- a/CodexDistTestCore/Timing.cs +++ /dev/null @@ -1,131 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class UseLongTimeoutsAttribute : PropertyAttribute - { - public UseLongTimeoutsAttribute() - : base(Timing.UseLongTimeoutsKey) - { - } - } - - public static class Timing - { - public const string UseLongTimeoutsKey = "UseLongTimeouts"; - - public static TimeSpan HttpCallTimeout() - { - return GetTimes().HttpCallTimeout(); - } - - public static int HttpCallRetryCount() - { - return GetTimes().HttpCallRetryCount(); - } - - public static void HttpCallRetryDelay() - { - Utils.Sleep(GetTimes().HttpCallRetryDelay()); - } - - public static void WaitForK8sServiceDelay() - { - Utils.Sleep(GetTimes().WaitForK8sServiceDelay()); - } - - public static TimeSpan K8sOperationTimeout() - { - return GetTimes().K8sOperationTimeout(); - } - - public static TimeSpan WaitForMetricTimeout() - { - return GetTimes().WaitForMetricTimeout(); - } - - private static ITimeSet GetTimes() - { - var testProperties = TestContext.CurrentContext.Test.Properties; - if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet(); - return new DefaultTimeSet(); - } - } - - public interface ITimeSet - { - TimeSpan HttpCallTimeout(); - int HttpCallRetryCount(); - TimeSpan HttpCallRetryDelay(); - TimeSpan WaitForK8sServiceDelay(); - TimeSpan K8sOperationTimeout(); - TimeSpan WaitForMetricTimeout(); - } - - public class DefaultTimeSet : ITimeSet - { - public TimeSpan HttpCallTimeout() - { - return TimeSpan.FromSeconds(10); - } - - public int HttpCallRetryCount() - { - return 5; - } - - public TimeSpan HttpCallRetryDelay() - { - return TimeSpan.FromSeconds(3); - } - - public TimeSpan WaitForK8sServiceDelay() - { - return TimeSpan.FromSeconds(1); - } - - public TimeSpan K8sOperationTimeout() - { - return TimeSpan.FromMinutes(5); - } - - public TimeSpan WaitForMetricTimeout() - { - return TimeSpan.FromSeconds(30); - } - } - - public class LongTimeSet : ITimeSet - { - public TimeSpan HttpCallTimeout() - { - return TimeSpan.FromHours(2); - } - - public int HttpCallRetryCount() - { - return 2; - } - - public TimeSpan HttpCallRetryDelay() - { - return TimeSpan.FromMinutes(5); - } - - public TimeSpan WaitForK8sServiceDelay() - { - return TimeSpan.FromSeconds(10); - } - - public TimeSpan K8sOperationTimeout() - { - return TimeSpan.FromMinutes(15); - } - - public TimeSpan WaitForMetricTimeout() - { - return TimeSpan.FromMinutes(5); - } - } -} diff --git a/CodexDistTestCore/TryContract.cs b/CodexDistTestCore/TryContract.cs deleted file mode 100644 index efc9d5a..0000000 --- a/CodexDistTestCore/TryContract.cs +++ /dev/null @@ -1,101 +0,0 @@ -using Nethereum.Web3; -using Nethereum.ABI.FunctionEncoding.Attributes; -using Nethereum.Contracts.CQS; -using Nethereum.Util; -using Nethereum.Web3.Accounts; -using Nethereum.Hex.HexConvertors.Extensions; -using Nethereum.Contracts; -using Nethereum.Contracts.Extensions; -using System.Numerics; -using NUnit.Framework; - -// https://docs.nethereum.com/en/latest/nethereum-smartcontrats-gettingstarted/ - -namespace CodexDistTestCore -{ - public class TryContract - { - [Test] - [Ignore("aaa")] - public void DoThing() - { - var url = "http://testchain.nethereum.com:8545"; - var privateKey = "0x7580e7fb49df1c861f0050fae31c2224c6aba908e116b8da44ee8cd927b990b0"; - var account = new Account(privateKey); - var web3 = new Web3(account, url); - - // Deploy contract: - var deploymentMessage = new StandardTokenDeployment - { - TotalSupply = 100000 - }; - var deploymentHandler = web3.Eth.GetContractDeploymentHandler(); - var transactionReceipt = Utils.Wait(deploymentHandler.SendRequestAndWaitForReceiptAsync(deploymentMessage)); - var contractAddress = transactionReceipt.ContractAddress; - - // Get balance: - var balanceOfFunctionMessage = new BalanceOfFunction() - { - Owner = account.Address, - }; - - var balanceHandler = web3.Eth.GetContractQueryHandler(); - var balance = Utils.Wait(balanceHandler.QueryAsync(contractAddress, balanceOfFunctionMessage)); - long asInt = ((long)balance); - - // Transfer: - var receiverAddress = "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"; - var transferHandler = web3.Eth.GetContractTransactionHandler(); - var transfer = new TransferFunction() - { - To = receiverAddress, - TokenAmount = 100 - }; - var transferReceipt = Utils.Wait(transferHandler.SendRequestAndWaitForReceiptAsync(contractAddress, transfer)); - - // Signing: - var signedTransaction = Utils.Wait(transferHandler.SignTransactionAsync(contractAddress, transfer)); - } - } - - public class StandardTokenDeployment : ContractDeploymentMessage - { - - public static string BYTECODE = "0x60606040526040516020806106f5833981016040528080519060200190919050505b80600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005081905550806000600050819055505b506106868061006f6000396000f360606040523615610074576000357c010000000000000000000000000000000000000000000000000000000090048063095ea7b31461008157806318160ddd146100b657806323b872dd146100d957806370a0823114610117578063a9059cbb14610143578063dd62ed3e1461017857610074565b61007f5b610002565b565b005b6100a060048080359060200190919080359060200190919050506101ad565b6040518082815260200191505060405180910390f35b6100c36004805050610674565b6040518082815260200191505060405180910390f35b6101016004808035906020019091908035906020019091908035906020019091905050610281565b6040518082815260200191505060405180910390f35b61012d600480803590602001909190505061048d565b6040518082815260200191505060405180910390f35b61016260048080359060200190919080359060200190919050506104cb565b6040518082815260200191505060405180910390f35b610197600480803590602001909190803590602001909190505061060b565b6040518082815260200191505060405180910390f35b600081600260005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008573ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925846040518082815260200191505060405180910390a36001905061027b565b92915050565b600081600160005060008673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561031b575081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505410155b80156103275750600082115b1561047c5781600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a381600160005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505403925050819055506001905061048656610485565b60009050610486565b5b9392505050565b6000600160005060008373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505490506104c6565b919050565b600081600160005060003373ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561050c5750600082115b156105fb5781600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a36001905061060556610604565b60009050610605565b5b92915050565b6000600260005060008473ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005054905061066e565b92915050565b60006000600050549050610683565b9056"; - - public StandardTokenDeployment() : base(BYTECODE) { } - - [Parameter("uint256", "totalSupply")] - public BigInteger TotalSupply { get; set; } - } - - [Function("balanceOf", "uint256")] - public class BalanceOfFunction : FunctionMessage - { - [Parameter("address", "_owner", 1)] - public string Owner { get; set; } - } - - [Function("transfer", "bool")] - public class TransferFunction : FunctionMessage - { - [Parameter("address", "_to", 1)] - public string To { get; set; } - - [Parameter("uint256", "_value", 2)] - public BigInteger TokenAmount { get; set; } - } - - [Event("Transfer")] - public class TransferEventDTO : IEventDTO - { - [Parameter("address", "_from", 1, true)] - public string From { get; set; } - - [Parameter("address", "_to", 2, true)] - public string To { get; set; } - - [Parameter("uint256", "_value", 3, false)] - public BigInteger Value { get; set; } - } -} diff --git a/CodexDistTestCore/Utils.cs b/CodexDistTestCore/Utils.cs deleted file mode 100644 index ae3e761..0000000 --- a/CodexDistTestCore/Utils.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace CodexDistTestCore -{ - public static class Utils - { - - } -} diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index a7f0c9b..3db5d59 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -1,4 +1,5 @@ -using CodexDistTestCore; +using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index c39069c..7acd78e 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -1,4 +1,5 @@ -using CodexDistTestCore; +using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests diff --git a/LongTests/TestsLong.csproj b/LongTests/TestsLong.csproj index fc5152f..136951d 100644 --- a/LongTests/TestsLong.csproj +++ b/LongTests/TestsLong.csproj @@ -13,7 +13,7 @@ - + diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 1f5c04f..4bc64fc 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -7,8 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\TestsLong.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" @@ -17,7 +15,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -33,10 +31,6 @@ Global {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Debug|Any CPU.Build.0 = Debug|Any CPU {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Release|Any CPU.ActiveCfg = Release|Any CPU {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Release|Any CPU.Build.0 = Release|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.Build.0 = Release|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.Build.0 = Debug|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.ActiveCfg = Release|Any CPU From 802f3459e9764513c859445de26e0920eac01d2b Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 09:10:39 +0200 Subject: [PATCH 19/21] Fixes issue where oneclient-test would fail because node was not ready. --- DistTestCore/Codex/CodexAccess.cs | 4 +--- DistTestCore/CodexNodeGroup.cs | 29 +++++++++++++++-------------- DistTestCore/FileManager.cs | 17 ++++++++++++----- Logging/FixtureLog.cs | 4 ++-- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index d0c0f72..c07a457 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -13,9 +13,7 @@ namespace DistTestCore.Codex public CodexDebugResponse GetDebugInfo() { - var response = Http().HttpGetJson("debug/info"); - //Log($"Got DebugInfo with id: '{response.id}'."); - return response; + return Http().HttpGetJson("debug/info"); } public string UploadFile(FileStream fileStream) diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 25d46c1..4e2cd16 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -47,13 +47,6 @@ namespace DistTestCore public RunningContainers Containers { get; private set; } public OnlineCodexNode[] Nodes { get; private set; } - //public GethCompanionGroup? GethCompanionGroup { get; set; } - - //public CodexNodeContainer[] GetContainers() - //{ - // return Nodes.Select(n => n.Container).ToArray(); - //} - public IEnumerator GetEnumerator() { return Nodes.Cast().GetEnumerator(); @@ -64,13 +57,6 @@ namespace DistTestCore return Nodes.GetEnumerator(); } - //public CodexNodeLog DownloadLog(IOnlineCodexNode node) - //{ - // var logDownloader = new PodLogDownloader(log, k8SManager); - // var n = (OnlineCodexNode)node; - // return logDownloader.DownloadLog(n); - //} - public string Describe() { return $""; @@ -79,7 +65,22 @@ namespace DistTestCore private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { var access = new CodexAccess(c); + EnsureOnline(access); return factory.CreateOnlineCodexNode(access, this); } + + private void EnsureOnline(CodexAccess access) + { + try + { + var debugInfo = access.GetDebugInfo(); + if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup."); + } + catch (Exception e) + { + lifecycle.Log.Error($"Failed to start codex node: {e}"); + throw; + } + } } } diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index 10f126b..b195e9c 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -14,7 +14,6 @@ namespace DistTestCore { public const int ChunkSize = 1024 * 1024; private readonly Random random = new Random(); - private readonly List activeFiles = new List(); private readonly TestLog log; private readonly string folder; @@ -22,7 +21,7 @@ namespace DistTestCore { folder = configuration.GetFileManagerFolder(); - if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + EnsureDirectory(); this.log = log; } @@ -30,7 +29,6 @@ namespace DistTestCore { var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); - activeFiles.Add(result); return result; } @@ -44,8 +42,7 @@ namespace DistTestCore public void DeleteAllTestFiles() { - foreach (var file in activeFiles) File.Delete(file.Filename); - activeFiles.Clear(); + DeleteDirectory(); } private void GenerateFileBytes(TestFile result, ByteSize size) @@ -66,6 +63,16 @@ namespace DistTestCore using var stream = new FileStream(result.Filename, FileMode.Append); stream.Write(bytes, 0, bytes.Length); } + + private void EnsureDirectory() + { + if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + } + + private void DeleteDirectory() + { + Directory.Delete(folder, true); + } } public class TestFile diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs index e06f4bc..7bb80ab 100644 --- a/Logging/FixtureLog.cs +++ b/Logging/FixtureLog.cs @@ -10,8 +10,8 @@ namespace Logging public FixtureLog(LogConfig config) { start = DateTime.UtcNow; - var folder = DetermineFolder(config); // "root/2023-04 /14" - var fixtureName = GetFixtureName(); // "11-09-23Z_ExampleTests" + var folder = DetermineFolder(config); + var fixtureName = GetFixtureName(); fullName = Path.Combine(folder, fixtureName); } From 8880ddd2bd8fc9acfcdc38c283c3b136e96e2729 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 10:31:14 +0200 Subject: [PATCH 20/21] Attempting to set up geth bootstrap argument --- DistTestCore/Logs/LogDownloadHandler.cs | 19 +++----- .../Marketplace/GethBootstrapNodeInfo.cs | 6 ++- .../Marketplace/GethBootstrapNodeStarter.cs | 6 ++- .../Marketplace/GethCompanionNodeStarter.cs | 2 +- .../Marketplace/GethContainerRecipe.cs | 15 ++++-- DistTestCore/Marketplace/GethInfoExtractor.cs | 48 +++++++++++++++++-- DistTestCore/Marketplace/GethStartupConfig.cs | 4 +- KubernetesWorkflow/K8sController.cs | 2 +- KubernetesWorkflow/StartupWorkflow.cs | 20 +++++++- Tests/BasicTests/ExampleTests.cs | 2 +- Tests/BasicTests/OneClientTests.cs | 1 - 11 files changed, 96 insertions(+), 29 deletions(-) diff --git a/DistTestCore/Logs/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs index a01ad9e..9cc1012 100644 --- a/DistTestCore/Logs/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -3,15 +3,16 @@ using Logging; namespace DistTestCore.Logs { - public class LogDownloadHandler : ILogHandler + public class LogDownloadHandler : LogHandler, ILogHandler { - private readonly string description; private readonly LogFile log; public LogDownloadHandler(string description, LogFile log) { - this.description = description; this.log = log; + + log.Write($"{description} -->> {log.FullFilename}"); + log.WriteRaw(description); } public CodexNodeLog CreateCodexNodeLog() @@ -19,17 +20,9 @@ namespace DistTestCore.Logs return new CodexNodeLog(log); } - public void Log(Stream stream) + protected override void ProcessLine(string line) { - log.Write($"{description} -->> {log.FullFilename}"); - log.WriteRaw(description); - var reader = new StreamReader(stream); - var line = reader.ReadLine(); - while (line != null) - { - log.WriteRaw(line); - line = reader.ReadLine(); - } + log.WriteRaw(line); } } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 39b6715..46189c4 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -6,16 +6,20 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeInfo { - public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64) + public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, Port discoveryPort) { RunningContainers = runningContainers; Account = account; GenesisJsonBase64 = genesisJsonBase64; + PubKey = pubKey; + DiscoveryPort = discoveryPort; } public RunningContainers RunningContainers { get; } public string Account { get; } public string GenesisJsonBase64 { get; } + public string PubKey { get; } + public Port DiscoveryPort { get; } public NethereumInteraction StartInteraction(TestLog log) { diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 7c114c4..2868752 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -26,16 +26,18 @@ namespace DistTestCore.Marketplace var extractor = new GethInfoExtractor(workflow, containers.Containers[0]); var account = extractor.ExtractAccount(); var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); + var pubKey = extractor.ExtractPubKey(); + var discoveryPort = containers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); Log($"Geth bootstrap node started with account '{account}'"); - return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64); + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort); } private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); - config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64)); + config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64, null!)); return config; } diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index b8b178b..c746bf1 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -38,7 +38,7 @@ namespace DistTestCore.Marketplace private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) { var config = new StartupConfig(); - config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64)); + config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64, bootstrapNode)); return config; } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 2dbfbde..fad225f 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -6,6 +6,7 @@ namespace DistTestCore.Marketplace { public const string DockerImage = "thatbenbierens/geth-confenv:latest"; public const string HttpPortTag = "http_port"; + public const string DiscoveryPortTag = "disc_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; @@ -23,18 +24,26 @@ namespace DistTestCore.Marketplace private string CreateArgs(GethStartupConfig config) { + var discovery = AddInternalPort(tag: DiscoveryPortTag); + if (config.IsBootstrapNode) { AddEnvVar("IS_BOOTSTRAP", "1"); var exposedPort = AddExposedPort(); - return $"--http.port {exposedPort.Number}"; + return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover"; } var port = AddInternalPort(); - var discovery = AddInternalPort(); var authRpc = AddInternalPort(); var httpPort = AddInternalPort(tag: HttpPortTag); - return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}"; + + var bootPubKey = config.BootstrapNode.PubKey; + var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip; + var bootPort = config.BootstrapNode.DiscoveryPort.Number; + var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}"; + // geth --bootnodes enode://pubkey1@ip1:port1 + + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --nodiscover {bootstrapArg}"; } } } diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/GethInfoExtractor.cs index f27e396..59725ec 100644 --- a/DistTestCore/Marketplace/GethInfoExtractor.cs +++ b/DistTestCore/Marketplace/GethInfoExtractor.cs @@ -17,7 +17,6 @@ namespace DistTestCore.Marketplace public string ExtractAccount() { var account = Retry(FetchAccount); - if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); return account; @@ -26,12 +25,17 @@ namespace DistTestCore.Marketplace public string ExtractGenesisJsonBase64() { var genesisJson = Retry(FetchGenesisJson); - if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure."); - var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + return Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + } - return encoded; + public string ExtractPubKey() + { + var pubKey = Retry(FetchPubKey); + if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure."); + + return pubKey; } private string Retry(Func fetch) @@ -54,5 +58,41 @@ namespace DistTestCore.Marketplace { return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); } + + private string FetchPubKey() + { + var enodeFinder = new PubKeyFinder(); + workflow.DownloadContainerLog(container, enodeFinder); + return enodeFinder.GetPubKey(); + } + } + + public class PubKeyFinder : LogHandler, ILogHandler + { + private const string openTag = "self=\"enode://"; + private string pubKey = string.Empty; + + public string GetPubKey() + { + return pubKey; + } + + protected override void ProcessLine(string line) + { + if (line.Contains(openTag)) + { + ExtractPubKey(line); + } + } + + private void ExtractPubKey(string line) + { + var openIndex = line.IndexOf(openTag) + openTag.Length; + var closeIndex = line.IndexOf("@"); + + pubKey = line.Substring( + startIndex: openIndex, + length: closeIndex - openIndex); + } } } diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs index 60164bb..baeb421 100644 --- a/DistTestCore/Marketplace/GethStartupConfig.cs +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -2,13 +2,15 @@ { public class GethStartupConfig { - public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64) + public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64, GethBootstrapNodeInfo bootstrapNode) { IsBootstrapNode = isBootstrapNode; GenesisJsonBase64 = genesisJsonBase64; + BootstrapNode = bootstrapNode; } public bool IsBootstrapNode { get; } public string GenesisJsonBase64 { get; } + public GethBootstrapNodeInfo BootstrapNode { get; } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 0804e8f..6ce366a 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -45,7 +45,7 @@ namespace KubernetesWorkflow public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { - var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); + using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); logHandler.Log(stream); } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index f575b15..d37699f 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -1,4 +1,6 @@ -namespace KubernetesWorkflow +using System.IO; + +namespace KubernetesWorkflow { public class StartupWorkflow { @@ -94,4 +96,20 @@ { void Log(Stream log); } + + public abstract class LogHandler : ILogHandler + { + public void Log(Stream log) + { + using var reader = new StreamReader(log); + var line = reader.ReadLine(); + while (line != null) + { + ProcessLine(line); + line = reader.ReadLine(); + } + } + + protected abstract void ProcessLine(string line); + } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index a3e3ae4..b70f3c4 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -49,7 +49,7 @@ namespace Tests.BasicTests [Test] public void MarketplaceExample() { - var group = SetupCodexNodes(4) + var group = SetupCodexNodes(2) .WithStorageQuota(10.GB()) .EnableMarketplace(initialBalance: 20) .BringOnline(); diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs index 76ff79b..a2c0c57 100644 --- a/Tests/BasicTests/OneClientTests.cs +++ b/Tests/BasicTests/OneClientTests.cs @@ -15,7 +15,6 @@ namespace Tests.BasicTests } [Test] - [Ignore("Unstable.")] public void RestartTest() { var group = SetupCodexNodes(1).BringOnline(); From ca822c508d69077d5b54555816bb237dcd16dbdc Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 11:28:07 +0200 Subject: [PATCH 21/21] Adds catch-retry to geth info extractor --- DistTestCore/Marketplace/GethInfoExtractor.cs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/GethInfoExtractor.cs index 59725ec..7151a60 100644 --- a/DistTestCore/Marketplace/GethInfoExtractor.cs +++ b/DistTestCore/Marketplace/GethInfoExtractor.cs @@ -40,7 +40,7 @@ namespace DistTestCore.Marketplace private string Retry(Func fetch) { - var result = fetch(); + var result = Catch(fetch); if (string.IsNullOrEmpty(result)) { Thread.Sleep(TimeSpan.FromSeconds(5)); @@ -49,6 +49,18 @@ namespace DistTestCore.Marketplace return result; } + private string Catch(Func fetch) + { + try + { + return fetch(); + } + catch + { + return string.Empty; + } + } + private string FetchGenesisJson() { return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename);