diff --git a/CodexDistTestCore/ByteSize.cs b/CodexDistTestCore/ByteSize.cs
deleted file mode 100644
index e8f5c92b..00000000
--- a/CodexDistTestCore/ByteSize.cs
+++ /dev/null
@@ -1,57 +0,0 @@
-namespace CodexDistTestCore
-{
- public class ByteSize
- {
- public ByteSize(long sizeInBytes)
- {
- SizeInBytes = sizeInBytes;
- }
-
- public long SizeInBytes { get; }
- }
-
- public static class IntExtensions
- {
- private const long Kilo = 1024;
-
- public static ByteSize KB(this long i)
- {
- return new ByteSize(i * Kilo);
- }
-
- public static ByteSize MB(this long i)
- {
- return (i * Kilo).KB();
- }
-
- public static ByteSize GB(this long i)
- {
- return (i * Kilo).MB();
- }
-
- public static ByteSize TB(this long i)
- {
- return (i * Kilo).GB();
- }
-
- public static ByteSize KB(this int i)
- {
- return Convert.ToInt64(i).KB();
- }
-
- public static ByteSize MB(this int i)
- {
- return Convert.ToInt64(i).MB();
- }
-
- public static ByteSize GB(this int i)
- {
- return Convert.ToInt64(i).GB();
- }
-
- public static ByteSize TB(this int i)
- {
- return Convert.ToInt64(i).TB();
- }
- }
-}
diff --git a/CodexDistTestCore/CodexAPI.cs b/CodexDistTestCore/CodexAPI.cs
deleted file mode 100644
index 997cd3c4..00000000
--- a/CodexDistTestCore/CodexAPI.cs
+++ /dev/null
@@ -1,17 +0,0 @@
-namespace CodexDistTestCore
-{
- public class CodexDebugResponse
- {
- public string id { get; set; } = string.Empty;
- public string[] addrs { get; set; } = new string[0];
- public string repo { get; set; } = string.Empty;
- public string spr { get; set; } = string.Empty;
- public CodexDebugVersionResponse codex { get; set; } = new();
- }
-
- public class CodexDebugVersionResponse
- {
- public string version { get; set; } = string.Empty;
- public string revision { get; set; } = string.Empty;
- }
-}
diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj
deleted file mode 100644
index cb1fb2ed..00000000
--- a/CodexDistTestCore/CodexDistTestCore.csproj
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
- net6.0
- CodexDistTestCore
- enable
- enable
-
-
-
-
-
-
-
-
-
diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs
deleted file mode 100644
index 71994ab9..00000000
--- a/CodexDistTestCore/CodexNodeContainer.cs
+++ /dev/null
@@ -1,89 +0,0 @@
-using CodexDistTestCore.Marketplace;
-
-namespace CodexDistTestCore
-{
- public class CodexNodeContainer
- {
- public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort)
- {
- Name = name;
- ServicePort = servicePort;
- ServicePortName = servicePortName;
- ApiPort = apiPort;
- ContainerPortName = containerPortName;
- DiscoveryPort = discoveryPort;
- ListenPort = listenPort;
- DataDir = dataDir;
- MetricsPort = metricsPort;
- }
-
- public string Name { get; }
- public int ServicePort { get; }
- public string ServicePortName { get; }
- public int ApiPort { get; }
- public string ContainerPortName { get; }
- public int DiscoveryPort { get; }
- public int ListenPort { get; }
- public string DataDir { get; }
- public int MetricsPort { get; }
-
- public GethCompanionNodeContainer? GethCompanionNodeContainer { get; set; } // :C
- }
-
- public class CodexGroupNumberSource
- {
- private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0);
- private readonly NumberSource groupContainerNameSource = new NumberSource(1);
- private readonly NumberSource servicePortSource = new NumberSource(30001);
-
- public int GetNextCodexNodeGroupNumber()
- {
- return codexNodeGroupNumberSource.GetNextNumber();
- }
-
- public string GetNextServicePortName()
- {
- return $"node{groupContainerNameSource.GetNextNumber()}";
- }
-
- public int GetNextServicePort()
- {
- return servicePortSource.GetNextNumber();
- }
- }
-
- public class CodexNodeContainerFactory
- {
- private readonly NumberSource containerNameSource = new NumberSource(1);
- private readonly NumberSource codexPortSource = new NumberSource(8080);
- private readonly CodexGroupNumberSource numberSource;
-
- public CodexNodeContainerFactory(CodexGroupNumberSource numberSource)
- {
- this.numberSource = numberSource;
- }
-
- public CodexNodeContainer CreateNext(OfflineCodexNodes offline)
- {
- var n = containerNameSource.GetNextNumber();
- return new CodexNodeContainer(
- name: $"codex-node{n}",
- servicePort: numberSource.GetNextServicePort(),
- servicePortName: numberSource.GetNextServicePortName(),
- apiPort: codexPortSource.GetNextNumber(),
- containerPortName: $"api-{n}",
- discoveryPort: codexPortSource.GetNextNumber(),
- listenPort: codexPortSource.GetNextNumber(),
- dataDir: $"datadir{n}",
- metricsPort: GetMetricsPort(offline)
- );
- }
-
- private int GetMetricsPort(OfflineCodexNodes offline)
- {
- if (offline.MetricsEnabled) return codexPortSource.GetNextNumber();
- return 0;
- }
-
- }
-}
diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs
deleted file mode 100644
index 12031aca..00000000
--- a/CodexDistTestCore/CodexNodeGroup.cs
+++ /dev/null
@@ -1,103 +0,0 @@
-using CodexDistTestCore.Config;
-using CodexDistTestCore.Marketplace;
-using k8s.Models;
-using System.Collections;
-
-namespace CodexDistTestCore
-{
- public interface ICodexNodeGroup : IEnumerable
- {
- IOfflineCodexNodes BringOffline();
- IOnlineCodexNode this[int index] { get; }
- }
-
- public class CodexNodeGroup : ICodexNodeGroup
- {
- private readonly TestLog log;
- private readonly IK8sManager k8SManager;
-
- public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
- {
- this.log = log;
- OrderNumber = orderNumber;
- Origin = origin;
- this.k8SManager = k8SManager;
- Nodes = nodes;
-
- foreach (var n in nodes) n.Group = this;
- }
-
- public IOnlineCodexNode this[int index]
- {
- get
- {
- return Nodes[index];
- }
- }
-
- public IOfflineCodexNodes BringOffline()
- {
- return k8SManager.BringOffline(this);
- }
-
- public int OrderNumber { get; }
- public OfflineCodexNodes Origin { get; }
- public OnlineCodexNode[] Nodes { get; }
- public V1Deployment? Deployment { get; set; }
- public V1Service? Service { get; set; }
- public PodInfo? PodInfo { get; set; }
- public GethCompanionGroup? GethCompanionGroup { get; set; }
-
- public CodexNodeContainer[] GetContainers()
- {
- return Nodes.Select(n => n.Container).ToArray();
- }
-
- public IEnumerator GetEnumerator()
- {
- return Nodes.Cast().GetEnumerator();
- }
-
- IEnumerator IEnumerable.GetEnumerator()
- {
- return Nodes.GetEnumerator();
- }
-
- public V1ObjectMeta GetServiceMetadata()
- {
- return new V1ObjectMeta
- {
- Name = "codex-test-entrypoint-" + OrderNumber,
- NamespaceProperty = K8sCluster.K8sNamespace
- };
- }
-
- public V1ObjectMeta GetDeploymentMetadata()
- {
- return new V1ObjectMeta
- {
- Name = "codex-test-node-" + OrderNumber,
- NamespaceProperty = K8sCluster.K8sNamespace
- };
- }
-
- public CodexNodeLog DownloadLog(IOnlineCodexNode node)
- {
- var logDownloader = new PodLogDownloader(log, k8SManager);
- var n = (OnlineCodexNode)node;
- return logDownloader.DownloadLog(n);
- }
-
- public Dictionary GetSelector()
- {
- return new Dictionary { { "codex-test-node", "dist-test-" + OrderNumber } };
- }
-
- public string Describe()
- {
- return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}";
- }
- }
-
-
-}
diff --git a/CodexDistTestCore/CodexNodeLog.cs b/CodexDistTestCore/CodexNodeLog.cs
deleted file mode 100644
index 1a0572af..00000000
--- a/CodexDistTestCore/CodexNodeLog.cs
+++ /dev/null
@@ -1,34 +0,0 @@
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
- public interface ICodexNodeLog
- {
- void AssertLogContains(string expectedString);
- }
-
- public class CodexNodeLog : ICodexNodeLog
- {
- private readonly LogFile logFile;
-
- public CodexNodeLog(LogFile logFile)
- {
- this.logFile = logFile;
- }
-
- public void AssertLogContains(string expectedString)
- {
- using var file = File.OpenRead(logFile.FullFilename);
- using var streamReader = new StreamReader(file);
-
- var line = streamReader.ReadLine();
- while (line != null)
- {
- if (line.Contains(expectedString)) return;
- line = streamReader.ReadLine();
- }
-
- Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}");
- }
- }
-}
diff --git a/CodexDistTestCore/Config/CodexDockerImage.cs b/CodexDistTestCore/Config/CodexDockerImage.cs
deleted file mode 100644
index a252a269..00000000
--- a/CodexDistTestCore/Config/CodexDockerImage.cs
+++ /dev/null
@@ -1,72 +0,0 @@
-using k8s.Models;
-
-namespace CodexDistTestCore.Config
-{
- public class CodexDockerImage
- {
- public string GetImageTag()
- {
- return "thatbenbierens/nim-codex:sha-b204837";
- }
-
- public string GetExpectedImageRevision()
- {
- return "b20483";
- }
-
- public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer container)
- {
- var formatter = new EnvFormatter();
- formatter.Create(node, container);
- return formatter.Result;
- }
-
- private class EnvFormatter
- {
- public List Result { get; } = new List();
-
- public void Create(OfflineCodexNodes node, CodexNodeContainer container)
- {
- AddVar("API_PORT", container.ApiPort.ToString());
- AddVar("DATA_DIR", container.DataDir);
- AddVar("DISC_PORT", container.DiscoveryPort.ToString());
- AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}");
-
- if (node.BootstrapNode != null)
- {
- var debugInfo = node.BootstrapNode.GetDebugInfo();
- AddVar("BOOTSTRAP_SPR", debugInfo.spr);
- }
- if (node.LogLevel != null)
- {
- AddVar("LOG_LEVEL", node.LogLevel.ToString()!.ToUpperInvariant());
- }
- if (node.StorageQuota != null)
- {
- AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!);
- }
- if (node.MetricsEnabled)
- {
- AddVar("METRICS_ADDR", "0.0.0.0");
- AddVar("METRICS_PORT", container.MetricsPort.ToString());
- }
- if (node.MarketplaceConfig != null)
- {
- //ETH_PROVIDER
- //ETH_ACCOUNT
- //ETH_DEPLOYMENT
- AddVar("ETH_ACCOUNT", container.GethCompanionNodeContainer!.Account);
- }
- }
-
- private void AddVar(string key, string value)
- {
- Result.Add(new V1EnvVar
- {
- Name = key,
- Value = value
- });
- }
- }
- }
-}
diff --git a/CodexDistTestCore/Config/FileManagerConfig.cs b/CodexDistTestCore/Config/FileManagerConfig.cs
deleted file mode 100644
index f7befc23..00000000
--- a/CodexDistTestCore/Config/FileManagerConfig.cs
+++ /dev/null
@@ -1,7 +0,0 @@
-namespace CodexDistTestCore.Config
-{
- public class FileManagerConfig
- {
- public const string Folder = "TestDataFiles";
- }
-}
diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs
deleted file mode 100644
index a17e6fdb..00000000
--- a/CodexDistTestCore/Config/K8sCluster.cs
+++ /dev/null
@@ -1,40 +0,0 @@
-using k8s;
-
-namespace CodexDistTestCore.Config
-{
- public class K8sCluster
- {
- public const string K8sNamespace = "";
- private const string KubeConfigFile = "C:\\kube\\config";
- private readonly Dictionary K8sNodeLocationMap = new Dictionary
- {
- { Location.BensLaptop, "worker01" },
- { Location.BensOldGamingMachine, "worker02" },
- };
-
- private KubernetesClientConfiguration? config;
-
- public KubernetesClientConfiguration GetK8sClientConfig()
- {
- if (config != null) return config;
- //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile);
- config = KubernetesClientConfiguration.BuildDefaultConfig();
- return config;
- }
-
- public string GetIp()
- {
- var c = GetK8sClientConfig();
-
- var host = c.Host.Replace("https://", "");
-
- return host.Substring(0, host.IndexOf(':'));
- }
-
- public string GetNodeLabelForLocation(Location location)
- {
- if (location == Location.Unspecified) return string.Empty;
- return K8sNodeLocationMap[location];
- }
- }
-}
diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs
deleted file mode 100644
index bf95aa7c..00000000
--- a/CodexDistTestCore/DistTest.cs
+++ /dev/null
@@ -1,120 +0,0 @@
-using CodexDistTestCore.Config;
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
- [SetUpFixture]
- public abstract class DistTest
- {
- private TestLog log = null!;
- private FileManager fileManager = null!;
- public K8sManager k8sManager = null!;
-
- [OneTimeSetUp]
- public void GlobalSetup()
- {
- // Previous test run may have been interrupted.
- // Begin by cleaning everything up.
- log = new TestLog();
- fileManager = new FileManager(log);
- k8sManager = new K8sManager(log, fileManager);
-
- try
- {
- k8sManager.DeleteAllResources();
- fileManager.DeleteAllTestFiles();
- }
- catch (Exception ex)
- {
- GlobalTestFailure.HasFailed = true;
- log.Error($"Global setup cleanup failed with: {ex}");
- throw;
- }
- log.Log("Global setup cleanup successful");
- }
-
- [SetUp]
- public void SetUpDistTest()
- {
- if (GlobalTestFailure.HasFailed)
- {
- Assert.Inconclusive("Skip test: Previous test failed during clean up.");
- }
- else
- {
- var dockerImage = new CodexDockerImage();
- log = new TestLog();
- log.Log($"Using docker image '{dockerImage.GetImageTag()}'");
-
- fileManager = new FileManager(log);
- k8sManager = new K8sManager(log, fileManager);
- }
- }
-
- [TearDown]
- public void TearDownDistTest()
- {
- try
- {
- log.EndTest();
- IncludeLogsAndMetricsOnTestFailure();
- k8sManager.DeleteAllResources();
- fileManager.DeleteAllTestFiles();
- }
- catch (Exception ex)
- {
- log.Error("Cleanup failed: " + ex.Message);
- GlobalTestFailure.HasFailed = true;
- }
- }
-
- public TestFile GenerateTestFile(ByteSize size)
- {
- return fileManager.GenerateTestFile(size);
- }
-
- public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes)
- {
- return new OfflineCodexNodes(k8sManager, numberOfNodes);
- }
-
- private void IncludeLogsAndMetricsOnTestFailure()
- {
- var result = TestContext.CurrentContext.Result;
- if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
- {
- if (IsDownloadingLogsAndMetricsEnabled())
- {
- log.Log("Downloading all CodexNode logs and metrics because of test failure...");
- k8sManager.ForEachOnlineGroup(DownloadLogs);
- k8sManager.DownloadAllMetrics();
- }
- else
- {
- log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
- }
- }
- }
-
- private void DownloadLogs(CodexNodeGroup group)
- {
- foreach (var node in group)
- {
- var downloader = new PodLogDownloader(log, k8sManager);
- var n = (OnlineCodexNode)node;
- downloader.DownloadLog(n);
- }
- }
-
- private bool IsDownloadingLogsAndMetricsEnabled()
- {
- var testProperties = TestContext.CurrentContext.Test.Properties;
- return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);
- }
- }
-
- public static class GlobalTestFailure
- {
- public static bool HasFailed { get; set; } = false;
- }
-}
diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs
deleted file mode 100644
index 6fbd55fa..00000000
--- a/CodexDistTestCore/FileManager.cs
+++ /dev/null
@@ -1,110 +0,0 @@
-using CodexDistTestCore.Config;
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
- public interface IFileManager
- {
- TestFile CreateEmptyTestFile();
- TestFile GenerateTestFile(ByteSize size);
- void DeleteAllTestFiles();
- }
-
- public class FileManager : IFileManager
- {
- public const int ChunkSize = 1024 * 1024;
- private readonly Random random = new Random();
- private readonly List activeFiles = new List();
- private readonly TestLog log;
-
- public FileManager(TestLog log)
- {
- if (!Directory.Exists(FileManagerConfig.Folder)) Directory.CreateDirectory(FileManagerConfig.Folder);
- this.log = log;
- }
-
- public TestFile CreateEmptyTestFile()
- {
- var result = new TestFile(Path.Combine(FileManagerConfig.Folder, Guid.NewGuid().ToString() + "_test.bin"));
- File.Create(result.Filename).Close();
- activeFiles.Add(result);
- return result;
- }
-
- public TestFile GenerateTestFile(ByteSize size)
- {
- var result = CreateEmptyTestFile();
- GenerateFileBytes(result, size);
- log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
- return result;
- }
-
- public void DeleteAllTestFiles()
- {
- foreach (var file in activeFiles) File.Delete(file.Filename);
- activeFiles.Clear();
- }
-
- private void GenerateFileBytes(TestFile result, ByteSize size)
- {
- long bytesLeft = size.SizeInBytes;
- while (bytesLeft > 0)
- {
- var length = Math.Min(bytesLeft, ChunkSize);
- AppendRandomBytesToFile(result, length);
- bytesLeft -= length;
- }
- }
-
- private void AppendRandomBytesToFile(TestFile result, long length)
- {
- var bytes = new byte[length];
- random.NextBytes(bytes);
- using var stream = new FileStream(result.Filename, FileMode.Append);
- stream.Write(bytes, 0, bytes.Length);
- }
- }
-
- public class TestFile
- {
- public TestFile(string filename)
- {
- Filename = filename;
- }
-
- public string Filename { get; }
-
- public long GetFileSize()
- {
- var info = new FileInfo(Filename);
- return info.Length;
- }
-
- public void AssertIsEqual(TestFile? actual)
- {
- if (actual == null) Assert.Fail("TestFile is null.");
- if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
-
- Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
-
- using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
- using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
-
- var bytesExpected = new byte[FileManager.ChunkSize];
- var bytesActual = new byte[FileManager.ChunkSize];
-
- var readExpected = 0;
- var readActual = 0;
-
- while (true)
- {
- readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
- readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
-
- if (readExpected == 0 && readActual == 0) return;
- Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
- CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal.");
- }
- }
- }
-}
diff --git a/CodexDistTestCore/Http.cs b/CodexDistTestCore/Http.cs
deleted file mode 100644
index fd7e31a8..00000000
--- a/CodexDistTestCore/Http.cs
+++ /dev/null
@@ -1,100 +0,0 @@
-using Newtonsoft.Json;
-using NUnit.Framework;
-using System.Net.Http.Headers;
-
-namespace CodexDistTestCore
-{
- public class Http
- {
- private readonly string ip;
- private readonly int port;
- private readonly string baseUrl;
-
- public Http(string ip, int port, string baseUrl)
- {
- this.ip = ip;
- this.port = port;
- this.baseUrl = baseUrl;
-
- if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
- if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
- }
-
- public string HttpGetString(string route)
- {
- return Retry(() =>
- {
- using var client = GetClient();
- var url = GetUrl() + route;
- var result = Utils.Wait(client.GetAsync(url));
- return Utils.Wait(result.Content.ReadAsStringAsync());
- });
- }
-
- public T HttpGetJson(string route)
- {
- return JsonConvert.DeserializeObject(HttpGetString(route))!;
- }
-
- public string HttpPostStream(string route, Stream stream)
- {
- return Retry(() =>
- {
- using var client = GetClient();
- var url = GetUrl() + route;
-
- var content = new StreamContent(stream);
- content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
- var response = Utils.Wait(client.PostAsync(url, content));
-
- return Utils.Wait(response.Content.ReadAsStringAsync());
- });
- }
-
- public Stream HttpGetStream(string route)
- {
- return Retry(() =>
- {
- var client = GetClient();
- var url = GetUrl() + route;
-
- return Utils.Wait(client.GetStreamAsync(url));
- });
- }
-
- private string GetUrl()
- {
- return $"http://{ip}:{port}{baseUrl}";
- }
-
- private static T Retry(Func operation)
- {
- var retryCounter = 0;
-
- while (true)
- {
- try
- {
- return operation();
- }
- catch (Exception exception)
- {
- Timing.HttpCallRetryDelay();
- retryCounter++;
- if (retryCounter > Timing.HttpCallRetryCount())
- {
- Assert.Fail(exception.Message);
- throw;
- }
- }
- }
- }
-
- private static HttpClient GetClient()
- {
- var client = new HttpClient();
- client.Timeout = Timing.HttpCallTimeout();
- return client;
- }
- }
-}
diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs
deleted file mode 100644
index 06bcafb5..00000000
--- a/CodexDistTestCore/K8sManager.cs
+++ /dev/null
@@ -1,177 +0,0 @@
-using CodexDistTestCore.Marketplace;
-using CodexDistTestCore.Metrics;
-
-namespace CodexDistTestCore
-{
- public interface IK8sManager
- {
- ICodexNodeGroup BringOnline(OfflineCodexNodes node);
- IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
- void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler);
- }
-
- public class K8sManager : IK8sManager
- {
- private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource();
- private readonly List onlineCodexNodeGroups = new List();
- private readonly KnownK8sPods knownPods = new KnownK8sPods();
- private readonly TestLog log;
- private readonly IFileManager fileManager;
- private readonly MetricsAggregator metricsAggregator;
- private readonly MarketplaceController marketplaceController;
-
- public K8sManager(TestLog log, IFileManager fileManager)
- {
- this.log = log;
- this.fileManager = fileManager;
- metricsAggregator = new MetricsAggregator(log, this);
- marketplaceController = new MarketplaceController(log, this);
- }
-
- public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
- {
- var group = CreateOnlineCodexNodes(offline);
-
- if (offline.MarketplaceConfig != null)
- {
- group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline);
- ConnectMarketplace(group);
- }
-
- K8s(k => k.BringOnline(group, offline));
-
- if (offline.MetricsEnabled)
- {
- BringOnlineMetrics(group);
- }
-
- log.Log($"{group.Describe()} online.");
-
- return group;
- }
-
- public IOfflineCodexNodes BringOffline(ICodexNodeGroup node)
- {
- var online = GetAndRemoveActiveNodeFor(node);
-
- K8s(k => k.BringOffline(online));
-
- log.Log($"{online.Describe()} offline.");
-
- return online.Origin;
- }
-
- public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments)
- {
- return K8s(k => k.ExecuteCommand(pod, containerName, command, arguments));
- }
-
- public void DeleteAllResources()
- {
- K8s(k => k.DeleteAllResources());
- }
-
- public void ForEachOnlineGroup(Action action)
- {
- foreach (var group in onlineCodexNodeGroups) action(group);
- }
-
- public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
- {
- K8s(k => k.FetchPodLog(node, logHandler));
- }
-
- public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber)
- {
- var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config);
-
- return K8s(k => k.BringOnlinePrometheus(spec));
- }
-
- public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec()
- {
- return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort());
- }
-
- public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
- {
- return K8s(k => k.BringOnlineGethBootstrapNode(spec));
- }
-
- public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group)
- {
- return K8s(k => k.BringOnlineGethCompanionGroup(info, group));
- }
-
- public void DownloadAllMetrics()
- {
- metricsAggregator.DownloadAllMetrics();
- }
-
- private void BringOnlineMetrics(CodexNodeGroup group)
- {
- metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group));
- }
-
- private void ConnectMarketplace(CodexNodeGroup group)
- {
- for (var i = 0; i < group.Nodes.Length; i++)
- {
- ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]);
- }
- }
-
- private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container)
- {
- node.Container.GethCompanionNodeContainer = container; // :c
-
- var access = new MarketplaceAccess(this, marketplaceController, log, group, container);
- access.Initialize();
- node.Marketplace = access;
- }
-
- private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
- {
- var containers = CreateContainers(offline);
- var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
- var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online);
- onlineCodexNodeGroups.Add(result);
- return result;
- }
-
- private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline)
- {
- var factory = new CodexNodeContainerFactory(codexGroupNumberSource);
- var containers = new List();
- for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline));
- return containers.ToArray();
- }
-
- private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
- {
- var n = (CodexNodeGroup)node;
- onlineCodexNodeGroups.Remove(n);
- return n;
- }
-
- private void K8s(Action action)
- {
- var k8s = new K8sOperations(knownPods);
- action(k8s);
- k8s.Close();
- }
-
- private T K8s(Func action)
- {
- var k8s = new K8sOperations(knownPods);
- var result = action(k8s);
- k8s.Close();
- return result;
- }
-
- private static OnlineCodexNode[] DowncastNodes(CodexNodeGroup group)
- {
- return group.Nodes.Cast().ToArray();
- }
- }
-}
diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs
deleted file mode 100644
index 0cc3ea1c..00000000
--- a/CodexDistTestCore/K8sOperations.cs
+++ /dev/null
@@ -1,349 +0,0 @@
-using CodexDistTestCore.Config;
-using CodexDistTestCore.Marketplace;
-using CodexDistTestCore.Metrics;
-using k8s;
-using k8s.Models;
-using NUnit.Framework;
-
-namespace CodexDistTestCore
-{
- public class K8sOperations
- {
- private readonly CodexDockerImage dockerImage = new CodexDockerImage();
- private readonly K8sCluster k8sCluster = new K8sCluster();
- private readonly Kubernetes client;
- private readonly KnownK8sPods knownPods;
-
- public K8sOperations(KnownK8sPods knownPods)
- {
- this.knownPods = knownPods;
-
- client = new Kubernetes(k8sCluster.GetK8sClientConfig());
- }
-
- public void Close()
- {
- client.Dispose();
- }
-
- public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline)
- {
- EnsureTestNamespace();
-
- CreateDeployment(online, offline);
- CreateService(online);
-
- WaitUntilOnline(online);
- FetchPodInfo(online);
- }
-
- public void BringOffline(CodexNodeGroup online)
- {
- var deploymentName = online.Deployment.Name();
- DeleteDeployment(online);
- DeleteService(online);
- WaitUntilOffline(deploymentName);
- }
-
- public void DeleteAllResources()
- {
- DeleteNamespace();
-
- WaitUntilZeroPods();
- WaitUntilNamespaceDeleted();
- }
-
- public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler)
- {
- var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name);
- logHandler.Log(stream);
- }
-
- public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments)
- {
- var runner = new CommandRunner(client, pod, containerName, command, arguments);
- runner.Run();
- return runner.GetStdOut();
- }
-
- public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec)
- {
- EnsureTestNamespace();
-
- CreatePrometheusDeployment(spec);
- CreatePrometheusService(spec);
- WaitUntilPrometheusOnline(spec);
-
- return new PrometheusInfo(spec.ServicePort, FetchNewPod());
- }
-
- public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec)
- {
- EnsureTestNamespace();
-
- CreateGethBootstrapDeployment(spec);
- CreateGethBootstrapService(spec);
- WaitUntilGethBootstrapOnline(spec);
-
- return FetchNewPod();
- }
-
- public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group)
- {
- EnsureTestNamespace();
-
- CreateGethCompanionDeployment(info, group);
- WaitUntilGethCompanionGroupOnline(info.Spec, group);
-
- return FetchNewPod();
- }
-
- private void FetchPodInfo(CodexNodeGroup online)
- {
- online.PodInfo = FetchNewPod();
- }
-
- private PodInfo FetchNewPod()
- {
- var pods = client.ListNamespacedPod(K8sNamespace).Items;
-
- var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
- Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure.");
-
- var newPod = newPods.Single();
- var info = new PodInfo(newPod.Name(), newPod.Status.PodIP);
-
- Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure.");
- Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure.");
-
- knownPods.Add(newPod.Name());
- return info;
- }
-
- #region Waiting
-
- private void WaitUntilOnline(CodexNodeGroup online)
- {
- WaitUntil(() =>
- {
- online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace);
- return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0;
- });
- }
-
- private void WaitUntilOffline(string deploymentName)
- {
- WaitUntil(() =>
- {
- var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
- return deployment == null || deployment.Status.AvailableReplicas == 0;
- });
- }
-
- private void WaitUntilZeroPods()
- {
- WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any());
- }
-
- private void WaitUntilNamespaceDeleted()
- {
- WaitUntil(() => !IsTestNamespaceOnline());
- }
-
- private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec)
- {
- WaitUntilDeploymentOnline(spec.GetDeploymentName());
- }
-
- private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec)
- {
- WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName());
- }
-
- private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group)
- {
- WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group));
- }
-
- private void WaitUntilDeploymentOnline(string deploymentName)
- {
- WaitUntil(() =>
- {
- var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
- return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
- });
- }
-
- private void WaitUntil(Func predicate)
- {
- var start = DateTime.UtcNow;
- var state = predicate();
- while (!state)
- {
- if (DateTime.UtcNow - start > Timing.K8sOperationTimeout())
- {
- Assert.Fail("K8s operation timed out.");
- throw new TimeoutException();
- }
-
- Timing.WaitForK8sServiceDelay();
- state = predicate();
- }
- }
-
- #endregion
-
- #region Service management
-
- private void CreateService(CodexNodeGroup online)
- {
- var serviceSpec = new V1Service
- {
- ApiVersion = "v1",
- Metadata = online.GetServiceMetadata(),
- Spec = new V1ServiceSpec
- {
- Type = "NodePort",
- Selector = online.GetSelector(),
- Ports = CreateServicePorts(online)
- }
- };
-
- online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
- }
-
- private List CreateServicePorts(CodexNodeGroup online)
- {
- var result = new List();
- var containers = online.GetContainers();
- foreach (var container in containers)
- {
- result.Add(new V1ServicePort
- {
- Name = container.ServicePortName,
- Protocol = "TCP",
- Port = container.ApiPort,
- TargetPort = container.ContainerPortName,
- NodePort = container.ServicePort
- });
- }
- return result;
- }
-
- private void DeleteService(CodexNodeGroup online)
- {
- if (online.Service == null) return;
- client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
- online.Service = null;
- }
-
- private void CreatePrometheusService(K8sPrometheusSpecs spec)
- {
- client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace);
- }
-
- private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec)
- {
- client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace);
- }
-
- #endregion
-
- #region Deployment management
-
- private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline)
- {
- var deploymentSpec = new V1Deployment
- {
- ApiVersion = "apps/v1",
- Metadata = online.GetDeploymentMetadata(),
- Spec = new V1DeploymentSpec
- {
- Replicas = 1,
- Selector = new V1LabelSelector
- {
- MatchLabels = online.GetSelector()
- },
- Template = new V1PodTemplateSpec
- {
- Metadata = new V1ObjectMeta
- {
- Labels = online.GetSelector()
- },
- Spec = new V1PodSpec
- {
- NodeSelector = CreateNodeSelector(offline),
- Containers = CreateDeploymentContainers(online, offline)
- }
- }
- }
- };
-
- online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
- }
-
- private IDictionary CreateNodeSelector(OfflineCodexNodes offline)
- {
- if (offline.Location == Location.Unspecified) return new Dictionary();
-
- return new Dictionary
- {
- { "codex-test-location", k8sCluster.GetNodeLabelForLocation(offline.Location) }
- };
- }
-
- private List CreateDeploymentContainers(CodexNodeGroup group, OfflineCodexNodes offline)
- {
- var result = new List();
- var containers = group.GetContainers();
- foreach (var container in containers)
- {
- result.Add(new V1Container
- {
- Name = container.Name,
- Image = dockerImage.GetImageTag(),
- Ports = new List
- {
- new V1ContainerPort
- {
- ContainerPort = container.ApiPort,
- Name = container.ContainerPortName
- }
- },
- Env = dockerImage.CreateEnvironmentVariables(offline, container)
- });
- }
-
- return result;
- }
-
- private void DeleteDeployment(CodexNodeGroup group)
- {
- if (group.Deployment == null) return;
- client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace);
- group.Deployment = null;
- }
-
- private void CreatePrometheusDeployment(K8sPrometheusSpecs spec)
- {
- client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace);
- }
-
- private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec)
- {
- client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace);
- }
-
- private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group)
- {
- client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace);
- }
-
- #endregion
-
- private class CommandRunner
- {
-
- }
- }
-}
diff --git a/CodexDistTestCore/KnownK8sPods.cs b/CodexDistTestCore/KnownK8sPods.cs
deleted file mode 100644
index 940a1472..00000000
--- a/CodexDistTestCore/KnownK8sPods.cs
+++ /dev/null
@@ -1,17 +0,0 @@
-namespace CodexDistTestCore
-{
- public class KnownK8sPods
- {
- private readonly List knownActivePodNames = new List();
-
- public bool Contains(string name)
- {
- return knownActivePodNames.Contains(name);
- }
-
- public void Add(string name)
- {
- knownActivePodNames.Add(name);
- }
- }
-}
diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs
deleted file mode 100644
index 74d687c0..00000000
--- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs
+++ /dev/null
@@ -1,35 +0,0 @@
-namespace CodexDistTestCore.Marketplace
-{
- public class GethCompanionGroup
- {
- public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers)
- {
- Number = number;
- Containers = containers;
- }
-
- public int Number { get; }
- public GethCompanionNodeContainer[] Containers { get; }
- public PodInfo? Pod { get; set; }
- }
-
- public class GethCompanionNodeContainer
- {
- public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName, int authRpcPort)
- {
- Name = name;
- ApiPort = apiPort;
- AuthRpcPort = authRpcPort;
- RpcPort = rpcPort;
- ContainerPortName = containerPortName;
- }
-
- public string Name { get; }
- public int ApiPort { get; }
- public int AuthRpcPort { get; }
- public int RpcPort { get; }
- public string ContainerPortName { get; }
-
- public string Account { get; set; } = string.Empty;
- }
-}
diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs
deleted file mode 100644
index bfaf3d4f..00000000
--- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs
+++ /dev/null
@@ -1,207 +0,0 @@
-using CodexDistTestCore.Config;
-using k8s.Models;
-
-namespace CodexDistTestCore.Marketplace
-{
- public static class GethDockerImage
- {
- public const string Image = "thatbenbierens/geth-confenv:latest";
-
- }
-
- public class K8sGethBoostrapSpecs
- {
- public const string ContainerName = "dtest-gethb";
- private const string portName = "gethb";
-
- public K8sGethBoostrapSpecs(int servicePort)
- {
- ServicePort = servicePort;
- }
-
- public int ServicePort { get; }
-
- public string GetBootstrapDeploymentName()
- {
- return "test-gethb";
- }
-
- public string GetCompanionDeploymentName(GethCompanionGroup group)
- {
- return "test-geth" + group.Number;
- }
-
- public V1Deployment CreateGethBootstrapDeployment()
- {
- var deploymentSpec = new V1Deployment
- {
- ApiVersion = "apps/v1",
- Metadata = new V1ObjectMeta
- {
- Name = GetBootstrapDeploymentName(),
- NamespaceProperty = K8sCluster.K8sNamespace
- },
- Spec = new V1DeploymentSpec
- {
- Replicas = 1,
- Selector = new V1LabelSelector
- {
- MatchLabels = CreateBootstrapSelector()
- },
- Template = new V1PodTemplateSpec
- {
- Metadata = new V1ObjectMeta
- {
- Labels = CreateBootstrapSelector()
- },
- Spec = new V1PodSpec
- {
- Containers = new List
- {
- new V1Container
- {
- Name = ContainerName,
- Image = GethDockerImage.Image,
- Ports = new List
- {
- new V1ContainerPort
- {
- ContainerPort = 8545,
- Name = portName
- }
- },
- Env = new List
- {
- new V1EnvVar
- {
- Name = "GETH_ARGS",
- Value = ""
- },
- new V1EnvVar
- {
- Name = "GENESIS_JSON",
- Value = genesisJsonBase64
- },
- new V1EnvVar
- {
- Name = "IS_BOOTSTRAP",
- Value = "1"
- }
- }
- }
- }
- }
- }
- }
- };
-
- return deploymentSpec;
- }
-
- public V1Service CreateGethBootstrapService()
- {
- var serviceSpec = new V1Service
- {
- ApiVersion = "v1",
- Metadata = new V1ObjectMeta
- {
- Name = "codex-gethb-service",
- NamespaceProperty = K8sCluster.K8sNamespace
- },
- Spec = new V1ServiceSpec
- {
- Type = "NodePort",
- Selector = CreateBootstrapSelector(),
- Ports = new List
- {
- new V1ServicePort
- {
- Name = "gethb-service",
- Protocol = "TCP",
- Port = 8545,
- TargetPort = portName,
- NodePort = ServicePort
- }
- }
- }
- };
-
- return serviceSpec;
- }
-
- public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info)
- {
- var deploymentSpec = new V1Deployment
- {
- ApiVersion = "apps/v1",
- Metadata = new V1ObjectMeta
- {
- Name = GetCompanionDeploymentName(group),
- NamespaceProperty = K8sCluster.K8sNamespace
- },
- Spec = new V1DeploymentSpec
- {
- Replicas = 1,
- Selector = new V1LabelSelector
- {
- MatchLabels = CreateCompanionSelector()
- },
- Template = new V1PodTemplateSpec
- {
- Metadata = new V1ObjectMeta
- {
- Labels = CreateCompanionSelector()
- },
- Spec = new V1PodSpec
- {
- Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList()
- }
- }
- }
- };
-
- return deploymentSpec;
- }
-
- private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info)
- {
- return new V1Container
- {
- Name = container.Name,
- Image = GethDockerImage.Image,
- Ports = new List
- {
- new V1ContainerPort
- {
- ContainerPort = container.ApiPort,
- Name = container.ContainerPortName
- }
- },
- // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64
- Env = new List
- {
- new V1EnvVar
- {
- Name = "GETH_ARGS",
- Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.AuthRpcPort} --http.port {container.RpcPort}"
- },
- new V1EnvVar
- {
- Name = "GENESIS_JSON",
- Value = info.GenesisJsonBase64
- }
- }
- };
- }
-
- private Dictionary CreateBootstrapSelector()
- {
- return new Dictionary { { "test-gethb", "dtest-gethb" } };
- }
-
- private Dictionary CreateCompanionSelector()
- {
- return new Dictionary { { "test-gethc", "dtest-gethc" } };
- }
- }
-}
diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs
deleted file mode 100644
index e9bf39e7..00000000
--- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs
+++ /dev/null
@@ -1,111 +0,0 @@
-using NUnit.Framework;
-using NUnit.Framework.Constraints;
-
-namespace CodexDistTestCore.Marketplace
-{
- public interface IMarketplaceAccess
- {
- void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral);
- void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes);
- void AssertThatBalance(IResolveConstraint constraint, string message = "");
- decimal GetBalance();
- }
-
- public class MarketplaceAccess : IMarketplaceAccess
- {
- private readonly K8sManager k8sManager;
- private readonly MarketplaceController marketplaceController;
- private readonly TestLog log;
- private readonly CodexNodeGroup group;
- private readonly GethCompanionNodeContainer container;
-
- public MarketplaceAccess(
- K8sManager k8sManager,
- MarketplaceController marketplaceController,
- TestLog log,
- CodexNodeGroup group,
- GethCompanionNodeContainer container)
- {
- this.k8sManager = k8sManager;
- this.marketplaceController = marketplaceController;
- this.log = log;
- this.group = group;
- this.container = container;
- }
-
- public void Initialize()
- {
- EnsureAccount();
-
- marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance);
-
- log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}");
- }
-
- public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
- {
- throw new NotImplementedException();
- }
-
- public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
- {
- throw new NotImplementedException();
- }
-
- public void AssertThatBalance(IResolveConstraint constraint, string message = "")
- {
- throw new NotImplementedException();
- }
-
- public decimal GetBalance()
- {
- return marketplaceController.GetBalance(container.Account);
- }
-
- private void EnsureAccount()
- {
- FetchAccount();
- if (string.IsNullOrEmpty(container.Account))
- {
- Thread.Sleep(TimeSpan.FromSeconds(15));
- FetchAccount();
- }
- Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure.");
- }
-
- private void FetchAccount()
- {
- container.Account = k8sManager.ExecuteCommand(group.GethCompanionGroup!.Pod!, container.Name, "cat", GethDockerImage.AccountFilename);
- }
- }
-
- public class MarketplaceUnavailable : IMarketplaceAccess
- {
- public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
- {
- Unavailable();
- }
-
- public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
- {
- Unavailable();
- }
-
- public void AssertThatBalance(IResolveConstraint constraint, string message = "")
- {
- Unavailable();
- }
-
- public decimal GetBalance()
- {
- Unavailable();
- return 0;
- }
-
- private void Unavailable()
- {
- Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");
- throw new InvalidOperationException();
- }
- }
-}
diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs
deleted file mode 100644
index 8bab9e93..00000000
--- a/CodexDistTestCore/Marketplace/MarketplaceController.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-using CodexDistTestCore.Config;
-using NUnit.Framework;
-using System.Numerics;
-using System.Text;
-
-namespace CodexDistTestCore.Marketplace
-{
- public class MarketplaceController
- {
- private readonly TestLog log;
- private readonly K8sManager k8sManager;
- private readonly NumberSource companionGroupNumberSource = new NumberSource(0);
- private List companionGroups = new List();
- private GethBootstrapInfo? bootstrapInfo;
-
- public MarketplaceController(TestLog log, K8sManager k8sManager)
- {
- this.log = log;
- this.k8sManager = k8sManager;
- }
-
-
- }
-}
diff --git a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs
deleted file mode 100644
index 23ad25f7..00000000
--- a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs
+++ /dev/null
@@ -1,4 +0,0 @@
-namespace CodexDistTestCore.Marketplace
-{
-
-}
diff --git a/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs b/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs
deleted file mode 100644
index 0fc0865c..00000000
--- a/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs
+++ /dev/null
@@ -1,122 +0,0 @@
-using CodexDistTestCore.Config;
-using k8s.Models;
-
-namespace CodexDistTestCore.Metrics
-{
- public class K8sPrometheusSpecs
- {
- public const string ContainerName = "dtest-prom";
- public const string ConfigFilepath = "/etc/prometheus/prometheus.yml";
- private const string dockerImage = "thatbenbierens/prometheus-envconf:latest";
- private const string portName = "prom-1";
- private readonly string config;
-
- public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config)
- {
- ServicePort = servicePort;
- PrometheusNumber = prometheusNumber;
- this.config = config;
- }
-
- public int ServicePort { get; }
- public int PrometheusNumber { get; }
-
- public string GetDeploymentName()
- {
- return "test-prom" + PrometheusNumber;
- }
-
- public V1Deployment CreatePrometheusDeployment()
- {
- var deploymentSpec = new V1Deployment
- {
- ApiVersion = "apps/v1",
- Metadata = new V1ObjectMeta
- {
- Name = GetDeploymentName(),
- NamespaceProperty = K8sCluster.K8sNamespace
- },
- Spec = new V1DeploymentSpec
- {
- Replicas = 1,
- Selector = new V1LabelSelector
- {
- MatchLabels = CreateSelector()
- },
- Template = new V1PodTemplateSpec
- {
- Metadata = new V1ObjectMeta
- {
- Labels = CreateSelector()
- },
- Spec = new V1PodSpec
- {
- Containers = new List
- {
- new V1Container
- {
- Name = ContainerName,
- Image = dockerImage,
- Ports = new List
- {
- new V1ContainerPort
- {
- ContainerPort = 9090,
- Name = portName
- }
- },
- Env = new List
- {
- new V1EnvVar
- {
- Name = "PROM_CONFIG",
- Value = config
- }
- }
- }
- }
- }
- }
- }
- };
-
- return deploymentSpec;
- }
-
- public V1Service CreatePrometheusService()
- {
- var serviceSpec = new V1Service
- {
- ApiVersion = "v1",
- Metadata = new V1ObjectMeta
- {
- Name = "codex-prom-service" + PrometheusNumber,
- NamespaceProperty = K8sCluster.K8sNamespace
- },
- Spec = new V1ServiceSpec
- {
- Type = "NodePort",
- Selector = CreateSelector(),
- Ports = new List
- {
- new V1ServicePort
- {
- Name = "prom-service" + PrometheusNumber,
- Protocol = "TCP",
- Port = 9090,
- TargetPort = portName,
- NodePort = ServicePort
- }
- }
- }
- };
-
- return serviceSpec;
- }
-
- private Dictionary CreateSelector()
- {
- return new Dictionary { { "test-prom", "dtest-prom" } };
- }
- }
-}
diff --git a/CodexDistTestCore/Metrics/MetricsAccess.cs b/CodexDistTestCore/Metrics/MetricsAccess.cs
deleted file mode 100644
index 2f6456e8..00000000
--- a/CodexDistTestCore/Metrics/MetricsAccess.cs
+++ /dev/null
@@ -1,63 +0,0 @@
-using NUnit.Framework;
-using NUnit.Framework.Constraints;
-
-namespace CodexDistTestCore.Metrics
-{
- public interface IMetricsAccess
- {
- void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
- }
-
- public class MetricsUnavailable : IMetricsAccess
- {
- public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
- {
- Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
- throw new InvalidOperationException();
- }
- }
-
- public class MetricsAccess : IMetricsAccess
- {
- private readonly MetricsQuery query;
- private readonly OnlineCodexNode node;
-
- public MetricsAccess(MetricsQuery query, OnlineCodexNode node)
- {
- this.query = query;
- this.node = node;
- }
-
- public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
- {
- var metricSet = GetMetricWithTimeout(metricName, node);
- var metricValue = metricSet.Values[0].Value;
- Assert.That(metricValue, constraint, message);
- }
-
- private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node)
- {
- var start = DateTime.UtcNow;
-
- while (true)
- {
- var mostRecent = GetMostRecent(metricName, node);
- if (mostRecent != null) return mostRecent;
- if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
- {
- Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
- throw new TimeoutException();
- }
-
- Utils.Sleep(TimeSpan.FromSeconds(2));
- }
- }
-
- private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node)
- {
- var result = query.GetMostRecent(metricName, node);
- if (result == null) return null;
- return result.Sets.LastOrDefault();
- }
- }
-}
diff --git a/CodexDistTestCore/Metrics/MetricsAggregator.cs b/CodexDistTestCore/Metrics/MetricsAggregator.cs
deleted file mode 100644
index adf0c3f6..00000000
--- a/CodexDistTestCore/Metrics/MetricsAggregator.cs
+++ /dev/null
@@ -1,78 +0,0 @@
-using NUnit.Framework;
-using System.Text;
-
-namespace CodexDistTestCore.Metrics
-{
- public class MetricsAggregator
- {
- private readonly NumberSource prometheusNumberSource = new NumberSource(0);
- private readonly TestLog log;
- private readonly K8sManager k8sManager;
- private readonly Dictionary activePrometheuses = new Dictionary();
-
- public MetricsAggregator(TestLog log, K8sManager k8sManager)
- {
- this.log = log;
- this.k8sManager = k8sManager;
- }
-
- public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
- {
- log.Log($"Starting metrics collecting for {nodes.Length} nodes...");
-
- var config = GeneratePrometheusConfig(nodes);
- var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber());
- var query = new MetricsQuery(prometheus);
- activePrometheuses.Add(query, nodes);
-
- log.Log("Metrics service started.");
-
- foreach (var node in nodes)
- {
- node.Metrics = new MetricsAccess(query, node);
- }
- }
-
- public void DownloadAllMetrics()
- {
- var download = new MetricsDownloader(log, activePrometheuses);
- download.DownloadAllMetrics();
- }
-
- private string GeneratePrometheusConfig(OnlineCodexNode[] nodes)
- {
- var config = "";
- config += "global:\n";
- config += " scrape_interval: 30s\n";
- config += " scrape_timeout: 10s\n";
- config += "\n";
- config += "scrape_configs:\n";
- config += " - job_name: services\n";
- config += " metrics_path: /metrics\n";
- config += " static_configs:\n";
- config += " - targets:\n";
-
- foreach (var node in nodes)
- {
- var ip = node.Group.PodInfo!.Ip;
- var port = node.Container.MetricsPort;
- config += $" - '{ip}:{port}'\n";
- }
-
- var bytes = Encoding.ASCII.GetBytes(config);
- return Convert.ToBase64String(bytes);
- }
- }
-
- public class PrometheusInfo
- {
- public PrometheusInfo(int servicePort, PodInfo podInfo)
- {
- ServicePort = servicePort;
- PodInfo = podInfo;
- }
-
- public int ServicePort { get; }
- public PodInfo PodInfo { get; }
- }
-}
diff --git a/CodexDistTestCore/Metrics/MetricsDownloader.cs b/CodexDistTestCore/Metrics/MetricsDownloader.cs
deleted file mode 100644
index 18fd10ba..00000000
--- a/CodexDistTestCore/Metrics/MetricsDownloader.cs
+++ /dev/null
@@ -1,97 +0,0 @@
-using System.Globalization;
-
-namespace CodexDistTestCore.Metrics
-{
- public class MetricsDownloader
- {
- private readonly TestLog log;
- private readonly Dictionary activePrometheuses;
-
- public MetricsDownloader(TestLog log, Dictionary activePrometheuses)
- {
- this.log = log;
- this.activePrometheuses = activePrometheuses;
- }
-
- public void DownloadAllMetrics()
- {
- foreach (var pair in activePrometheuses)
- {
- DownloadAllMetrics(pair.Key, pair.Value);
- }
- }
-
- private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes)
- {
- foreach (var node in nodes)
- {
- DownloadAllMetricsForNode(query, node);
- }
- }
-
- private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node)
- {
- var metrics = query.GetAllMetricsForNode(node);
- if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
-
- var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
- var map = CreateValueMap(metrics);
-
- WriteToFile(node.GetName(), headers, map);
- }
-
- private void WriteToFile(string nodeName, string[] headers, Dictionary> map)
- {
- var file = log.CreateSubfile("csv");
- log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}");
-
- file.WriteRaw(string.Join(",", headers));
-
- foreach (var pair in map)
- {
- file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
- }
- }
-
- private Dictionary> CreateValueMap(Metrics metrics)
- {
- var map = CreateForAllTimestamps(metrics);
- foreach (var metric in metrics.Sets)
- {
- AddToMap(map, metric);
- }
- return map;
-
- }
-
- private Dictionary> CreateForAllTimestamps(Metrics metrics)
- {
- var result = new Dictionary>();
- var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
- foreach (var timestamp in timestamps) result.Add(timestamp, new List());
- return result;
- }
-
- private void AddToMap(Dictionary> map, MetricsSet metric)
- {
- foreach (var key in map.Keys)
- {
- map[key].Add(GetValueAtTimestamp(key, metric));
- }
- }
-
- private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
- {
- var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
- if (value == null) return "";
- return value.Value.ToString(CultureInfo.InvariantCulture);
- }
-
- private string FormatTimestamp(DateTime key)
- {
- var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
- var diff = key - origin;
- return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
- }
- }
-}
diff --git a/CodexDistTestCore/Metrics/MetricsQuery.cs b/CodexDistTestCore/Metrics/MetricsQuery.cs
deleted file mode 100644
index c028a6c2..00000000
--- a/CodexDistTestCore/Metrics/MetricsQuery.cs
+++ /dev/null
@@ -1,190 +0,0 @@
-using CodexDistTestCore.Config;
-using System.Globalization;
-
-namespace CodexDistTestCore.Metrics
-{
- public class MetricsQuery
- {
- private readonly K8sCluster k8sCluster = new K8sCluster();
- private readonly Http http;
-
- public MetricsQuery(PrometheusInfo prometheusInfo)
- {
- http = new Http(
- k8sCluster.GetIp(),
- prometheusInfo.ServicePort,
- "api/v1");
- }
-
- public Metrics? GetMostRecent(string metricName, OnlineCodexNode node)
- {
- var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
- if (response == null) return null;
-
- return new Metrics
- {
- Sets = response.data.result.Select(r =>
- {
- return new MetricsSet
- {
- Instance = r.metric.instance,
- Values = MapSingleValue(r.value)
- };
- }).ToArray()
- };
- }
-
- public Metrics? GetMetrics(string metricName)
- {
- var response = GetAll(metricName);
- if (response == null) return null;
- return MapResponseToMetrics(response);
- }
-
- public Metrics? GetAllMetricsForNode(OnlineCodexNode node)
- {
- var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
- if (response.status != "success") return null;
- return MapResponseToMetrics(response);
- }
-
- private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
- {
- var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
- if (response.status != "success") return null;
- return response;
- }
-
- private PrometheusQueryResponse? GetAll(string metricName)
- {
- var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}");
- if (response.status != "success") return null;
- return response;
- }
-
- private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
- {
- return new Metrics
- {
- Sets = response.data.result.Select(r =>
- {
- return new MetricsSet
- {
- Name = r.metric.__name__,
- Instance = r.metric.instance,
- Values = MapMultipleValues(r.values)
- };
- }).ToArray()
- };
- }
-
- private MetricsSetValue[] MapSingleValue(object[] value)
- {
- if (value != null && value.Length > 0)
- {
- return new[]
- {
- MapValue(value)
- };
- }
- return Array.Empty();
- }
-
- private MetricsSetValue[] MapMultipleValues(object[][] values)
- {
- if (values != null && values.Length > 0)
- {
- return values.Select(v => MapValue(v)).ToArray();
- }
- return Array.Empty();
- }
-
- private MetricsSetValue MapValue(object[] value)
- {
- if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
-
- return new MetricsSetValue
- {
- Timestamp = ToTimestamp(value[0]),
- Value = ToValue(value[1])
- };
- }
-
- private string GetInstanceNameForNode(OnlineCodexNode node)
- {
- var pod = node.Group.PodInfo!;
- return $"{pod.Ip}:{node.Container.MetricsPort}";
- }
-
- private string GetInstanceStringForNode(OnlineCodexNode node)
- {
- return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
- }
-
- private string GetQueryTimeRange()
- {
- return "[12h]";
- }
-
- private double ToValue(object v)
- {
- return Convert.ToDouble(v, CultureInfo.InvariantCulture);
- }
-
- private DateTime ToTimestamp(object v)
- {
- var unixSeconds = ToValue(v);
- return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
- }
- }
-
- public class Metrics
- {
- public MetricsSet[] Sets { get; set; } = Array.Empty();
- }
-
- public class MetricsSet
- {
- public string Name { get; set; } = string.Empty;
- public string Instance { get; set; } = string.Empty;
- public MetricsSetValue[] Values { get; set; } = Array.Empty();
- }
-
- public class MetricsSetValue
- {
- public DateTime Timestamp { get; set; }
- public double Value { get; set; }
- }
-
- public class PrometheusQueryResponse
- {
- public string status { get; set; } = string.Empty;
- public PrometheusQueryResponseData data { get; set; } = new();
- }
-
- public class PrometheusQueryResponseData
- {
- public string resultType { get; set; } = string.Empty;
- public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty();
- }
-
- public class PrometheusQueryResponseDataResultEntry
- {
- public ResultEntryMetric metric { get; set; } = new();
- public object[] value { get; set; } = Array.Empty