From 3f159b8ece92fed6dfa56ef7860e7c463d011f58 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 07:56:08 +0200 Subject: [PATCH] Removes old backend --- CodexDistTestCore/ByteSize.cs | 57 --- CodexDistTestCore/CodexAPI.cs | 17 - CodexDistTestCore/CodexDistTestCore.csproj | 16 - CodexDistTestCore/CodexNodeContainer.cs | 89 ----- CodexDistTestCore/CodexNodeGroup.cs | 103 ------ CodexDistTestCore/CodexNodeLog.cs | 34 -- CodexDistTestCore/Config/CodexDockerImage.cs | 72 ---- CodexDistTestCore/Config/FileManagerConfig.cs | 7 - CodexDistTestCore/Config/K8sCluster.cs | 40 -- CodexDistTestCore/DistTest.cs | 120 ------ CodexDistTestCore/FileManager.cs | 110 ------ CodexDistTestCore/Http.cs | 100 ----- CodexDistTestCore/K8sManager.cs | 177 --------- CodexDistTestCore/K8sOperations.cs | 349 ------------------ CodexDistTestCore/KnownK8sPods.cs | 17 - .../Marketplace/GethCompanionNodeContainer.cs | 35 -- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 207 ----------- .../Marketplace/MarketplaceAccess.cs | 111 ------ .../Marketplace/MarketplaceController.cs | 24 -- .../Marketplace/MarketplaceInitialConfig.cs | 4 - .../Metrics/K8sPrometheusSpecs.cs | 122 ------ CodexDistTestCore/Metrics/MetricsAccess.cs | 63 ---- .../Metrics/MetricsAggregator.cs | 78 ---- .../Metrics/MetricsDownloader.cs | 97 ----- CodexDistTestCore/Metrics/MetricsQuery.cs | 190 ---------- CodexDistTestCore/NumberSource.cs | 19 - CodexDistTestCore/OfflineCodexNodes.cs | 97 ----- CodexDistTestCore/OnlineCodexNode.cs | 141 ------- CodexDistTestCore/PodLogDownloader.cs | 64 ---- CodexDistTestCore/Timing.cs | 131 ------- CodexDistTestCore/TryContract.cs | 101 ----- CodexDistTestCore/Utils.cs | 7 - LongTests/BasicTests/LargeFileTests.cs | 3 +- LongTests/BasicTests/TestInfraTests.cs | 3 +- LongTests/TestsLong.csproj | 2 +- cs-codex-dist-testing.sln | 8 +- 36 files changed, 6 insertions(+), 2809 deletions(-) delete mode 100644 CodexDistTestCore/ByteSize.cs delete mode 100644 CodexDistTestCore/CodexAPI.cs delete mode 100644 CodexDistTestCore/CodexDistTestCore.csproj delete mode 100644 CodexDistTestCore/CodexNodeContainer.cs delete mode 100644 CodexDistTestCore/CodexNodeGroup.cs delete mode 100644 CodexDistTestCore/CodexNodeLog.cs delete mode 100644 CodexDistTestCore/Config/CodexDockerImage.cs delete mode 100644 CodexDistTestCore/Config/FileManagerConfig.cs delete mode 100644 CodexDistTestCore/Config/K8sCluster.cs delete mode 100644 CodexDistTestCore/DistTest.cs delete mode 100644 CodexDistTestCore/FileManager.cs delete mode 100644 CodexDistTestCore/Http.cs delete mode 100644 CodexDistTestCore/K8sManager.cs delete mode 100644 CodexDistTestCore/K8sOperations.cs delete mode 100644 CodexDistTestCore/KnownK8sPods.cs delete mode 100644 CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs delete mode 100644 CodexDistTestCore/Marketplace/K8sGethSpecs.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceAccess.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceController.cs delete mode 100644 CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs delete mode 100644 CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsAccess.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsAggregator.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsDownloader.cs delete mode 100644 CodexDistTestCore/Metrics/MetricsQuery.cs delete mode 100644 CodexDistTestCore/NumberSource.cs delete mode 100644 CodexDistTestCore/OfflineCodexNodes.cs delete mode 100644 CodexDistTestCore/OnlineCodexNode.cs delete mode 100644 CodexDistTestCore/PodLogDownloader.cs delete mode 100644 CodexDistTestCore/Timing.cs delete mode 100644 CodexDistTestCore/TryContract.cs delete mode 100644 CodexDistTestCore/Utils.cs diff --git a/CodexDistTestCore/ByteSize.cs b/CodexDistTestCore/ByteSize.cs deleted file mode 100644 index e8f5c92b..00000000 --- a/CodexDistTestCore/ByteSize.cs +++ /dev/null @@ -1,57 +0,0 @@ -namespace CodexDistTestCore -{ - public class ByteSize - { - public ByteSize(long sizeInBytes) - { - SizeInBytes = sizeInBytes; - } - - public long SizeInBytes { get; } - } - - public static class IntExtensions - { - private const long Kilo = 1024; - - public static ByteSize KB(this long i) - { - return new ByteSize(i * Kilo); - } - - public static ByteSize MB(this long i) - { - return (i * Kilo).KB(); - } - - public static ByteSize GB(this long i) - { - return (i * Kilo).MB(); - } - - public static ByteSize TB(this long i) - { - return (i * Kilo).GB(); - } - - public static ByteSize KB(this int i) - { - return Convert.ToInt64(i).KB(); - } - - public static ByteSize MB(this int i) - { - return Convert.ToInt64(i).MB(); - } - - public static ByteSize GB(this int i) - { - return Convert.ToInt64(i).GB(); - } - - public static ByteSize TB(this int i) - { - return Convert.ToInt64(i).TB(); - } - } -} diff --git a/CodexDistTestCore/CodexAPI.cs b/CodexDistTestCore/CodexAPI.cs deleted file mode 100644 index 997cd3c4..00000000 --- a/CodexDistTestCore/CodexAPI.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace CodexDistTestCore -{ - public class CodexDebugResponse - { - public string id { get; set; } = string.Empty; - public string[] addrs { get; set; } = new string[0]; - public string repo { get; set; } = string.Empty; - public string spr { get; set; } = string.Empty; - public CodexDebugVersionResponse codex { get; set; } = new(); - } - - public class CodexDebugVersionResponse - { - public string version { get; set; } = string.Empty; - public string revision { get; set; } = string.Empty; - } -} diff --git a/CodexDistTestCore/CodexDistTestCore.csproj b/CodexDistTestCore/CodexDistTestCore.csproj deleted file mode 100644 index cb1fb2ed..00000000 --- a/CodexDistTestCore/CodexDistTestCore.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net6.0 - CodexDistTestCore - enable - enable - - - - - - - - - diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs deleted file mode 100644 index 71994ab9..00000000 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ /dev/null @@ -1,89 +0,0 @@ -using CodexDistTestCore.Marketplace; - -namespace CodexDistTestCore -{ - public class CodexNodeContainer - { - public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort) - { - Name = name; - ServicePort = servicePort; - ServicePortName = servicePortName; - ApiPort = apiPort; - ContainerPortName = containerPortName; - DiscoveryPort = discoveryPort; - ListenPort = listenPort; - DataDir = dataDir; - MetricsPort = metricsPort; - } - - public string Name { get; } - public int ServicePort { get; } - public string ServicePortName { get; } - public int ApiPort { get; } - public string ContainerPortName { get; } - public int DiscoveryPort { get; } - public int ListenPort { get; } - public string DataDir { get; } - public int MetricsPort { get; } - - public GethCompanionNodeContainer? GethCompanionNodeContainer { get; set; } // :C - } - - public class CodexGroupNumberSource - { - private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0); - private readonly NumberSource groupContainerNameSource = new NumberSource(1); - private readonly NumberSource servicePortSource = new NumberSource(30001); - - public int GetNextCodexNodeGroupNumber() - { - return codexNodeGroupNumberSource.GetNextNumber(); - } - - public string GetNextServicePortName() - { - return $"node{groupContainerNameSource.GetNextNumber()}"; - } - - public int GetNextServicePort() - { - return servicePortSource.GetNextNumber(); - } - } - - public class CodexNodeContainerFactory - { - private readonly NumberSource containerNameSource = new NumberSource(1); - private readonly NumberSource codexPortSource = new NumberSource(8080); - private readonly CodexGroupNumberSource numberSource; - - public CodexNodeContainerFactory(CodexGroupNumberSource numberSource) - { - this.numberSource = numberSource; - } - - public CodexNodeContainer CreateNext(OfflineCodexNodes offline) - { - var n = containerNameSource.GetNextNumber(); - return new CodexNodeContainer( - name: $"codex-node{n}", - servicePort: numberSource.GetNextServicePort(), - servicePortName: numberSource.GetNextServicePortName(), - apiPort: codexPortSource.GetNextNumber(), - containerPortName: $"api-{n}", - discoveryPort: codexPortSource.GetNextNumber(), - listenPort: codexPortSource.GetNextNumber(), - dataDir: $"datadir{n}", - metricsPort: GetMetricsPort(offline) - ); - } - - private int GetMetricsPort(OfflineCodexNodes offline) - { - if (offline.MetricsEnabled) return codexPortSource.GetNextNumber(); - return 0; - } - - } -} diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs deleted file mode 100644 index 12031aca..00000000 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ /dev/null @@ -1,103 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using k8s.Models; -using System.Collections; - -namespace CodexDistTestCore -{ - public interface ICodexNodeGroup : IEnumerable - { - IOfflineCodexNodes BringOffline(); - IOnlineCodexNode this[int index] { get; } - } - - public class CodexNodeGroup : ICodexNodeGroup - { - private readonly TestLog log; - private readonly IK8sManager k8SManager; - - public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) - { - this.log = log; - OrderNumber = orderNumber; - Origin = origin; - this.k8SManager = k8SManager; - Nodes = nodes; - - foreach (var n in nodes) n.Group = this; - } - - public IOnlineCodexNode this[int index] - { - get - { - return Nodes[index]; - } - } - - public IOfflineCodexNodes BringOffline() - { - return k8SManager.BringOffline(this); - } - - public int OrderNumber { get; } - public OfflineCodexNodes Origin { get; } - public OnlineCodexNode[] Nodes { get; } - public V1Deployment? Deployment { get; set; } - public V1Service? Service { get; set; } - public PodInfo? PodInfo { get; set; } - public GethCompanionGroup? GethCompanionGroup { get; set; } - - public CodexNodeContainer[] GetContainers() - { - return Nodes.Select(n => n.Container).ToArray(); - } - - public IEnumerator GetEnumerator() - { - return Nodes.Cast().GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return Nodes.GetEnumerator(); - } - - public V1ObjectMeta GetServiceMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-entrypoint-" + OrderNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }; - } - - public V1ObjectMeta GetDeploymentMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-node-" + OrderNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }; - } - - public CodexNodeLog DownloadLog(IOnlineCodexNode node) - { - var logDownloader = new PodLogDownloader(log, k8SManager); - var n = (OnlineCodexNode)node; - return logDownloader.DownloadLog(n); - } - - public Dictionary GetSelector() - { - return new Dictionary { { "codex-test-node", "dist-test-" + OrderNumber } }; - } - - public string Describe() - { - return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}"; - } - } - - -} diff --git a/CodexDistTestCore/CodexNodeLog.cs b/CodexDistTestCore/CodexNodeLog.cs deleted file mode 100644 index 1a0572af..00000000 --- a/CodexDistTestCore/CodexNodeLog.cs +++ /dev/null @@ -1,34 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface ICodexNodeLog - { - void AssertLogContains(string expectedString); - } - - public class CodexNodeLog : ICodexNodeLog - { - private readonly LogFile logFile; - - public CodexNodeLog(LogFile logFile) - { - this.logFile = logFile; - } - - public void AssertLogContains(string expectedString) - { - using var file = File.OpenRead(logFile.FullFilename); - using var streamReader = new StreamReader(file); - - var line = streamReader.ReadLine(); - while (line != null) - { - if (line.Contains(expectedString)) return; - line = streamReader.ReadLine(); - } - - Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}"); - } - } -} diff --git a/CodexDistTestCore/Config/CodexDockerImage.cs b/CodexDistTestCore/Config/CodexDockerImage.cs deleted file mode 100644 index a252a269..00000000 --- a/CodexDistTestCore/Config/CodexDockerImage.cs +++ /dev/null @@ -1,72 +0,0 @@ -using k8s.Models; - -namespace CodexDistTestCore.Config -{ - public class CodexDockerImage - { - public string GetImageTag() - { - return "thatbenbierens/nim-codex:sha-b204837"; - } - - public string GetExpectedImageRevision() - { - return "b20483"; - } - - public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer container) - { - var formatter = new EnvFormatter(); - formatter.Create(node, container); - return formatter.Result; - } - - private class EnvFormatter - { - public List Result { get; } = new List(); - - public void Create(OfflineCodexNodes node, CodexNodeContainer container) - { - AddVar("API_PORT", container.ApiPort.ToString()); - AddVar("DATA_DIR", container.DataDir); - AddVar("DISC_PORT", container.DiscoveryPort.ToString()); - AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{container.ListenPort}"); - - if (node.BootstrapNode != null) - { - var debugInfo = node.BootstrapNode.GetDebugInfo(); - AddVar("BOOTSTRAP_SPR", debugInfo.spr); - } - if (node.LogLevel != null) - { - AddVar("LOG_LEVEL", node.LogLevel.ToString()!.ToUpperInvariant()); - } - if (node.StorageQuota != null) - { - AddVar("STORAGE_QUOTA", node.StorageQuota.SizeInBytes.ToString()!); - } - if (node.MetricsEnabled) - { - AddVar("METRICS_ADDR", "0.0.0.0"); - AddVar("METRICS_PORT", container.MetricsPort.ToString()); - } - if (node.MarketplaceConfig != null) - { - //ETH_PROVIDER - //ETH_ACCOUNT - //ETH_DEPLOYMENT - AddVar("ETH_ACCOUNT", container.GethCompanionNodeContainer!.Account); - } - } - - private void AddVar(string key, string value) - { - Result.Add(new V1EnvVar - { - Name = key, - Value = value - }); - } - } - } -} diff --git a/CodexDistTestCore/Config/FileManagerConfig.cs b/CodexDistTestCore/Config/FileManagerConfig.cs deleted file mode 100644 index f7befc23..00000000 --- a/CodexDistTestCore/Config/FileManagerConfig.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace CodexDistTestCore.Config -{ - public class FileManagerConfig - { - public const string Folder = "TestDataFiles"; - } -} diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs deleted file mode 100644 index a17e6fdb..00000000 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ /dev/null @@ -1,40 +0,0 @@ -using k8s; - -namespace CodexDistTestCore.Config -{ - public class K8sCluster - { - public const string K8sNamespace = ""; - private const string KubeConfigFile = "C:\\kube\\config"; - private readonly Dictionary K8sNodeLocationMap = new Dictionary - { - { Location.BensLaptop, "worker01" }, - { Location.BensOldGamingMachine, "worker02" }, - }; - - private KubernetesClientConfiguration? config; - - public KubernetesClientConfiguration GetK8sClientConfig() - { - if (config != null) return config; - //config = KubernetesClientConfiguration.BuildConfigFromConfigFile(KubeConfigFile); - config = KubernetesClientConfiguration.BuildDefaultConfig(); - return config; - } - - public string GetIp() - { - var c = GetK8sClientConfig(); - - var host = c.Host.Replace("https://", ""); - - return host.Substring(0, host.IndexOf(':')); - } - - public string GetNodeLabelForLocation(Location location) - { - if (location == Location.Unspecified) return string.Empty; - return K8sNodeLocationMap[location]; - } - } -} diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs deleted file mode 100644 index bf95aa7c..00000000 --- a/CodexDistTestCore/DistTest.cs +++ /dev/null @@ -1,120 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - [SetUpFixture] - public abstract class DistTest - { - private TestLog log = null!; - private FileManager fileManager = null!; - public K8sManager k8sManager = null!; - - [OneTimeSetUp] - public void GlobalSetup() - { - // Previous test run may have been interrupted. - // Begin by cleaning everything up. - log = new TestLog(); - fileManager = new FileManager(log); - k8sManager = new K8sManager(log, fileManager); - - try - { - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); - } - catch (Exception ex) - { - GlobalTestFailure.HasFailed = true; - log.Error($"Global setup cleanup failed with: {ex}"); - throw; - } - log.Log("Global setup cleanup successful"); - } - - [SetUp] - public void SetUpDistTest() - { - if (GlobalTestFailure.HasFailed) - { - Assert.Inconclusive("Skip test: Previous test failed during clean up."); - } - else - { - var dockerImage = new CodexDockerImage(); - log = new TestLog(); - log.Log($"Using docker image '{dockerImage.GetImageTag()}'"); - - fileManager = new FileManager(log); - k8sManager = new K8sManager(log, fileManager); - } - } - - [TearDown] - public void TearDownDistTest() - { - try - { - log.EndTest(); - IncludeLogsAndMetricsOnTestFailure(); - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); - } - catch (Exception ex) - { - log.Error("Cleanup failed: " + ex.Message); - GlobalTestFailure.HasFailed = true; - } - } - - public TestFile GenerateTestFile(ByteSize size) - { - return fileManager.GenerateTestFile(size); - } - - public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes) - { - return new OfflineCodexNodes(k8sManager, numberOfNodes); - } - - private void IncludeLogsAndMetricsOnTestFailure() - { - var result = TestContext.CurrentContext.Result; - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { - if (IsDownloadingLogsAndMetricsEnabled()) - { - log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - k8sManager.ForEachOnlineGroup(DownloadLogs); - k8sManager.DownloadAllMetrics(); - } - else - { - log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - } - } - } - - private void DownloadLogs(CodexNodeGroup group) - { - foreach (var node in group) - { - var downloader = new PodLogDownloader(log, k8sManager); - var n = (OnlineCodexNode)node; - downloader.DownloadLog(n); - } - } - - private bool IsDownloadingLogsAndMetricsEnabled() - { - var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); - } - } - - public static class GlobalTestFailure - { - public static bool HasFailed { get; set; } = false; - } -} diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs deleted file mode 100644 index 6fbd55fa..00000000 --- a/CodexDistTestCore/FileManager.cs +++ /dev/null @@ -1,110 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IFileManager - { - TestFile CreateEmptyTestFile(); - TestFile GenerateTestFile(ByteSize size); - void DeleteAllTestFiles(); - } - - public class FileManager : IFileManager - { - public const int ChunkSize = 1024 * 1024; - private readonly Random random = new Random(); - private readonly List activeFiles = new List(); - private readonly TestLog log; - - public FileManager(TestLog log) - { - if (!Directory.Exists(FileManagerConfig.Folder)) Directory.CreateDirectory(FileManagerConfig.Folder); - this.log = log; - } - - public TestFile CreateEmptyTestFile() - { - var result = new TestFile(Path.Combine(FileManagerConfig.Folder, Guid.NewGuid().ToString() + "_test.bin")); - File.Create(result.Filename).Close(); - activeFiles.Add(result); - return result; - } - - public TestFile GenerateTestFile(ByteSize size) - { - var result = CreateEmptyTestFile(); - GenerateFileBytes(result, size); - log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); - return result; - } - - public void DeleteAllTestFiles() - { - foreach (var file in activeFiles) File.Delete(file.Filename); - activeFiles.Clear(); - } - - private void GenerateFileBytes(TestFile result, ByteSize size) - { - long bytesLeft = size.SizeInBytes; - while (bytesLeft > 0) - { - var length = Math.Min(bytesLeft, ChunkSize); - AppendRandomBytesToFile(result, length); - bytesLeft -= length; - } - } - - private void AppendRandomBytesToFile(TestFile result, long length) - { - var bytes = new byte[length]; - random.NextBytes(bytes); - using var stream = new FileStream(result.Filename, FileMode.Append); - stream.Write(bytes, 0, bytes.Length); - } - } - - public class TestFile - { - public TestFile(string filename) - { - Filename = filename; - } - - public string Filename { get; } - - public long GetFileSize() - { - var info = new FileInfo(Filename); - return info.Length; - } - - public void AssertIsEqual(TestFile? actual) - { - if (actual == null) Assert.Fail("TestFile is null."); - if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); - - Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); - - using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); - using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); - - var bytesExpected = new byte[FileManager.ChunkSize]; - var bytesActual = new byte[FileManager.ChunkSize]; - - var readExpected = 0; - var readActual = 0; - - while (true) - { - readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); - readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); - - if (readExpected == 0 && readActual == 0) return; - Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); - CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); - } - } - } -} diff --git a/CodexDistTestCore/Http.cs b/CodexDistTestCore/Http.cs deleted file mode 100644 index fd7e31a8..00000000 --- a/CodexDistTestCore/Http.cs +++ /dev/null @@ -1,100 +0,0 @@ -using Newtonsoft.Json; -using NUnit.Framework; -using System.Net.Http.Headers; - -namespace CodexDistTestCore -{ - public class Http - { - private readonly string ip; - private readonly int port; - private readonly string baseUrl; - - public Http(string ip, int port, string baseUrl) - { - this.ip = ip; - this.port = port; - this.baseUrl = baseUrl; - - if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; - if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; - } - - public string HttpGetString(string route) - { - return Retry(() => - { - using var client = GetClient(); - var url = GetUrl() + route; - var result = Utils.Wait(client.GetAsync(url)); - return Utils.Wait(result.Content.ReadAsStringAsync()); - }); - } - - public T HttpGetJson(string route) - { - return JsonConvert.DeserializeObject(HttpGetString(route))!; - } - - public string HttpPostStream(string route, Stream stream) - { - return Retry(() => - { - using var client = GetClient(); - var url = GetUrl() + route; - - var content = new StreamContent(stream); - content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - var response = Utils.Wait(client.PostAsync(url, content)); - - return Utils.Wait(response.Content.ReadAsStringAsync()); - }); - } - - public Stream HttpGetStream(string route) - { - return Retry(() => - { - var client = GetClient(); - var url = GetUrl() + route; - - return Utils.Wait(client.GetStreamAsync(url)); - }); - } - - private string GetUrl() - { - return $"http://{ip}:{port}{baseUrl}"; - } - - private static T Retry(Func operation) - { - var retryCounter = 0; - - while (true) - { - try - { - return operation(); - } - catch (Exception exception) - { - Timing.HttpCallRetryDelay(); - retryCounter++; - if (retryCounter > Timing.HttpCallRetryCount()) - { - Assert.Fail(exception.Message); - throw; - } - } - } - } - - private static HttpClient GetClient() - { - var client = new HttpClient(); - client.Timeout = Timing.HttpCallTimeout(); - return client; - } - } -} diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs deleted file mode 100644 index 06bcafb5..00000000 --- a/CodexDistTestCore/K8sManager.cs +++ /dev/null @@ -1,177 +0,0 @@ -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; - -namespace CodexDistTestCore -{ - public interface IK8sManager - { - ICodexNodeGroup BringOnline(OfflineCodexNodes node); - IOfflineCodexNodes BringOffline(ICodexNodeGroup node); - void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler); - } - - public class K8sManager : IK8sManager - { - private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource(); - private readonly List onlineCodexNodeGroups = new List(); - private readonly KnownK8sPods knownPods = new KnownK8sPods(); - private readonly TestLog log; - private readonly IFileManager fileManager; - private readonly MetricsAggregator metricsAggregator; - private readonly MarketplaceController marketplaceController; - - public K8sManager(TestLog log, IFileManager fileManager) - { - this.log = log; - this.fileManager = fileManager; - metricsAggregator = new MetricsAggregator(log, this); - marketplaceController = new MarketplaceController(log, this); - } - - public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) - { - var group = CreateOnlineCodexNodes(offline); - - if (offline.MarketplaceConfig != null) - { - group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline); - ConnectMarketplace(group); - } - - K8s(k => k.BringOnline(group, offline)); - - if (offline.MetricsEnabled) - { - BringOnlineMetrics(group); - } - - log.Log($"{group.Describe()} online."); - - return group; - } - - public IOfflineCodexNodes BringOffline(ICodexNodeGroup node) - { - var online = GetAndRemoveActiveNodeFor(node); - - K8s(k => k.BringOffline(online)); - - log.Log($"{online.Describe()} offline."); - - return online.Origin; - } - - public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) - { - return K8s(k => k.ExecuteCommand(pod, containerName, command, arguments)); - } - - public void DeleteAllResources() - { - K8s(k => k.DeleteAllResources()); - } - - public void ForEachOnlineGroup(Action action) - { - foreach (var group in onlineCodexNodeGroups) action(group); - } - - public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler) - { - K8s(k => k.FetchPodLog(node, logHandler)); - } - - public PrometheusInfo BringOnlinePrometheus(string config, int prometheusNumber) - { - var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config); - - return K8s(k => k.BringOnlinePrometheus(spec)); - } - - public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec() - { - return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); - } - - public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) - { - return K8s(k => k.BringOnlineGethBootstrapNode(spec)); - } - - public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) - { - return K8s(k => k.BringOnlineGethCompanionGroup(info, group)); - } - - public void DownloadAllMetrics() - { - metricsAggregator.DownloadAllMetrics(); - } - - private void BringOnlineMetrics(CodexNodeGroup group) - { - metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group)); - } - - private void ConnectMarketplace(CodexNodeGroup group) - { - for (var i = 0; i < group.Nodes.Length; i++) - { - ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]); - } - } - - private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container) - { - node.Container.GethCompanionNodeContainer = container; // :c - - var access = new MarketplaceAccess(this, marketplaceController, log, group, container); - access.Initialize(); - node.Marketplace = access; - } - - private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) - { - var containers = CreateContainers(offline); - var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); - var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); - onlineCodexNodeGroups.Add(result); - return result; - } - - private CodexNodeContainer[] CreateContainers(OfflineCodexNodes offline) - { - var factory = new CodexNodeContainerFactory(codexGroupNumberSource); - var containers = new List(); - for (var i = 0; i < offline.NumberOfNodes; i++) containers.Add(factory.CreateNext(offline)); - return containers.ToArray(); - } - - private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node) - { - var n = (CodexNodeGroup)node; - onlineCodexNodeGroups.Remove(n); - return n; - } - - private void K8s(Action action) - { - var k8s = new K8sOperations(knownPods); - action(k8s); - k8s.Close(); - } - - private T K8s(Func action) - { - var k8s = new K8sOperations(knownPods); - var result = action(k8s); - k8s.Close(); - return result; - } - - private static OnlineCodexNode[] DowncastNodes(CodexNodeGroup group) - { - return group.Nodes.Cast().ToArray(); - } - } -} diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs deleted file mode 100644 index 0cc3ea1c..00000000 --- a/CodexDistTestCore/K8sOperations.cs +++ /dev/null @@ -1,349 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; -using k8s; -using k8s.Models; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public class K8sOperations - { - private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly Kubernetes client; - private readonly KnownK8sPods knownPods; - - public K8sOperations(KnownK8sPods knownPods) - { - this.knownPods = knownPods; - - client = new Kubernetes(k8sCluster.GetK8sClientConfig()); - } - - public void Close() - { - client.Dispose(); - } - - public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) - { - EnsureTestNamespace(); - - CreateDeployment(online, offline); - CreateService(online); - - WaitUntilOnline(online); - FetchPodInfo(online); - } - - public void BringOffline(CodexNodeGroup online) - { - var deploymentName = online.Deployment.Name(); - DeleteDeployment(online); - DeleteService(online); - WaitUntilOffline(deploymentName); - } - - public void DeleteAllResources() - { - DeleteNamespace(); - - WaitUntilZeroPods(); - WaitUntilNamespaceDeleted(); - } - - public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler) - { - var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name); - logHandler.Log(stream); - } - - public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) - { - var runner = new CommandRunner(client, pod, containerName, command, arguments); - runner.Run(); - return runner.GetStdOut(); - } - - public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec) - { - EnsureTestNamespace(); - - CreatePrometheusDeployment(spec); - CreatePrometheusService(spec); - WaitUntilPrometheusOnline(spec); - - return new PrometheusInfo(spec.ServicePort, FetchNewPod()); - } - - public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) - { - EnsureTestNamespace(); - - CreateGethBootstrapDeployment(spec); - CreateGethBootstrapService(spec); - WaitUntilGethBootstrapOnline(spec); - - return FetchNewPod(); - } - - public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) - { - EnsureTestNamespace(); - - CreateGethCompanionDeployment(info, group); - WaitUntilGethCompanionGroupOnline(info.Spec, group); - - return FetchNewPod(); - } - - private void FetchPodInfo(CodexNodeGroup online) - { - online.PodInfo = FetchNewPod(); - } - - private PodInfo FetchNewPod() - { - var pods = client.ListNamespacedPod(K8sNamespace).Items; - - var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); - Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); - - var newPod = newPods.Single(); - var info = new PodInfo(newPod.Name(), newPod.Status.PodIP); - - Assert.That(!string.IsNullOrEmpty(info.Name), "Invalid pod name received. Test infra failure."); - Assert.That(!string.IsNullOrEmpty(info.Ip), "Invalid pod IP received. Test infra failure."); - - knownPods.Add(newPod.Name()); - return info; - } - - #region Waiting - - private void WaitUntilOnline(CodexNodeGroup online) - { - WaitUntil(() => - { - online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; - }); - } - - private void WaitUntilOffline(string deploymentName) - { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment == null || deployment.Status.AvailableReplicas == 0; - }); - } - - private void WaitUntilZeroPods() - { - WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); - } - - private void WaitUntilNamespaceDeleted() - { - WaitUntil(() => !IsTestNamespaceOnline()); - } - - private void WaitUntilPrometheusOnline(K8sPrometheusSpecs spec) - { - WaitUntilDeploymentOnline(spec.GetDeploymentName()); - } - - private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) - { - WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); - } - - private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) - { - WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); - } - - private void WaitUntilDeploymentOnline(string deploymentName) - { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; - }); - } - - private void WaitUntil(Func predicate) - { - var start = DateTime.UtcNow; - var state = predicate(); - while (!state) - { - if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) - { - Assert.Fail("K8s operation timed out."); - throw new TimeoutException(); - } - - Timing.WaitForK8sServiceDelay(); - state = predicate(); - } - } - - #endregion - - #region Service management - - private void CreateService(CodexNodeGroup online) - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = online.GetServiceMetadata(), - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = online.GetSelector(), - Ports = CreateServicePorts(online) - } - }; - - online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); - } - - private List CreateServicePorts(CodexNodeGroup online) - { - var result = new List(); - var containers = online.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1ServicePort - { - Name = container.ServicePortName, - Protocol = "TCP", - Port = container.ApiPort, - TargetPort = container.ContainerPortName, - NodePort = container.ServicePort - }); - } - return result; - } - - private void DeleteService(CodexNodeGroup online) - { - if (online.Service == null) return; - client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); - online.Service = null; - } - - private void CreatePrometheusService(K8sPrometheusSpecs spec) - { - client.CreateNamespacedService(spec.CreatePrometheusService(), K8sNamespace); - } - - private void CreateGethBootstrapService(K8sGethBoostrapSpecs spec) - { - client.CreateNamespacedService(spec.CreateGethBootstrapService(), K8sNamespace); - } - - #endregion - - #region Deployment management - - private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = online.GetDeploymentMetadata(), - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = online.GetSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = online.GetSelector() - }, - Spec = new V1PodSpec - { - NodeSelector = CreateNodeSelector(offline), - Containers = CreateDeploymentContainers(online, offline) - } - } - } - }; - - online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - } - - private IDictionary CreateNodeSelector(OfflineCodexNodes offline) - { - if (offline.Location == Location.Unspecified) return new Dictionary(); - - return new Dictionary - { - { "codex-test-location", k8sCluster.GetNodeLabelForLocation(offline.Location) } - }; - } - - private List CreateDeploymentContainers(CodexNodeGroup group, OfflineCodexNodes offline) - { - var result = new List(); - var containers = group.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1Container - { - Name = container.Name, - Image = dockerImage.GetImageTag(), - Ports = new List - { - new V1ContainerPort - { - ContainerPort = container.ApiPort, - Name = container.ContainerPortName - } - }, - Env = dockerImage.CreateEnvironmentVariables(offline, container) - }); - } - - return result; - } - - private void DeleteDeployment(CodexNodeGroup group) - { - if (group.Deployment == null) return; - client.DeleteNamespacedDeployment(group.Deployment.Name(), K8sNamespace); - group.Deployment = null; - } - - private void CreatePrometheusDeployment(K8sPrometheusSpecs spec) - { - client.CreateNamespacedDeployment(spec.CreatePrometheusDeployment(), K8sNamespace); - } - - private void CreateGethBootstrapDeployment(K8sGethBoostrapSpecs spec) - { - client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); - } - - private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) - { - client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); - } - - #endregion - - private class CommandRunner - { - - } - } -} diff --git a/CodexDistTestCore/KnownK8sPods.cs b/CodexDistTestCore/KnownK8sPods.cs deleted file mode 100644 index 940a1472..00000000 --- a/CodexDistTestCore/KnownK8sPods.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace CodexDistTestCore -{ - public class KnownK8sPods - { - private readonly List knownActivePodNames = new List(); - - public bool Contains(string name) - { - return knownActivePodNames.Contains(name); - } - - public void Add(string name) - { - knownActivePodNames.Add(name); - } - } -} diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs deleted file mode 100644 index 74d687c0..00000000 --- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace CodexDistTestCore.Marketplace -{ - public class GethCompanionGroup - { - public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers) - { - Number = number; - Containers = containers; - } - - public int Number { get; } - public GethCompanionNodeContainer[] Containers { get; } - public PodInfo? Pod { get; set; } - } - - public class GethCompanionNodeContainer - { - public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName, int authRpcPort) - { - Name = name; - ApiPort = apiPort; - AuthRpcPort = authRpcPort; - RpcPort = rpcPort; - ContainerPortName = containerPortName; - } - - public string Name { get; } - public int ApiPort { get; } - public int AuthRpcPort { get; } - public int RpcPort { get; } - public string ContainerPortName { get; } - - public string Account { get; set; } = string.Empty; - } -} diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs deleted file mode 100644 index bfaf3d4f..00000000 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ /dev/null @@ -1,207 +0,0 @@ -using CodexDistTestCore.Config; -using k8s.Models; - -namespace CodexDistTestCore.Marketplace -{ - public static class GethDockerImage - { - public const string Image = "thatbenbierens/geth-confenv:latest"; - - } - - public class K8sGethBoostrapSpecs - { - public const string ContainerName = "dtest-gethb"; - private const string portName = "gethb"; - - public K8sGethBoostrapSpecs(int servicePort) - { - ServicePort = servicePort; - } - - public int ServicePort { get; } - - public string GetBootstrapDeploymentName() - { - return "test-gethb"; - } - - public string GetCompanionDeploymentName(GethCompanionGroup group) - { - return "test-geth" + group.Number; - } - - public V1Deployment CreateGethBootstrapDeployment() - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetBootstrapDeploymentName(), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateBootstrapSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateBootstrapSelector() - }, - Spec = new V1PodSpec - { - Containers = new List - { - new V1Container - { - Name = ContainerName, - Image = GethDockerImage.Image, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 8545, - Name = portName - } - }, - Env = new List - { - new V1EnvVar - { - Name = "GETH_ARGS", - Value = "" - }, - new V1EnvVar - { - Name = "GENESIS_JSON", - Value = genesisJsonBase64 - }, - new V1EnvVar - { - Name = "IS_BOOTSTRAP", - Value = "1" - } - } - } - } - } - } - } - }; - - return deploymentSpec; - } - - public V1Service CreateGethBootstrapService() - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = "codex-gethb-service", - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = CreateBootstrapSelector(), - Ports = new List - { - new V1ServicePort - { - Name = "gethb-service", - Protocol = "TCP", - Port = 8545, - TargetPort = portName, - NodePort = ServicePort - } - } - } - }; - - return serviceSpec; - } - - public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetCompanionDeploymentName(group), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateCompanionSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateCompanionSelector() - }, - Spec = new V1PodSpec - { - Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList() - } - } - } - }; - - return deploymentSpec; - } - - private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info) - { - return new V1Container - { - Name = container.Name, - Image = GethDockerImage.Image, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = container.ApiPort, - Name = container.ContainerPortName - } - }, - // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 - Env = new List - { - new V1EnvVar - { - Name = "GETH_ARGS", - Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.AuthRpcPort} --http.port {container.RpcPort}" - }, - new V1EnvVar - { - Name = "GENESIS_JSON", - Value = info.GenesisJsonBase64 - } - } - }; - } - - private Dictionary CreateBootstrapSelector() - { - return new Dictionary { { "test-gethb", "dtest-gethb" } }; - } - - private Dictionary CreateCompanionSelector() - { - return new Dictionary { { "test-gethc", "dtest-gethc" } }; - } - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs deleted file mode 100644 index e9bf39e7..00000000 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ /dev/null @@ -1,111 +0,0 @@ -using NUnit.Framework; -using NUnit.Framework.Constraints; - -namespace CodexDistTestCore.Marketplace -{ - public interface IMarketplaceAccess - { - void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral); - void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes); - void AssertThatBalance(IResolveConstraint constraint, string message = ""); - decimal GetBalance(); - } - - public class MarketplaceAccess : IMarketplaceAccess - { - private readonly K8sManager k8sManager; - private readonly MarketplaceController marketplaceController; - private readonly TestLog log; - private readonly CodexNodeGroup group; - private readonly GethCompanionNodeContainer container; - - public MarketplaceAccess( - K8sManager k8sManager, - MarketplaceController marketplaceController, - TestLog log, - CodexNodeGroup group, - GethCompanionNodeContainer container) - { - this.k8sManager = k8sManager; - this.marketplaceController = marketplaceController; - this.log = log; - this.group = group; - this.container = container; - } - - public void Initialize() - { - EnsureAccount(); - - marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); - - log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); - } - - public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) - { - throw new NotImplementedException(); - } - - public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) - { - throw new NotImplementedException(); - } - - public void AssertThatBalance(IResolveConstraint constraint, string message = "") - { - throw new NotImplementedException(); - } - - public decimal GetBalance() - { - return marketplaceController.GetBalance(container.Account); - } - - private void EnsureAccount() - { - FetchAccount(); - if (string.IsNullOrEmpty(container.Account)) - { - Thread.Sleep(TimeSpan.FromSeconds(15)); - FetchAccount(); - } - Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); - } - - private void FetchAccount() - { - container.Account = k8sManager.ExecuteCommand(group.GethCompanionGroup!.Pod!, container.Name, "cat", GethDockerImage.AccountFilename); - } - } - - public class MarketplaceUnavailable : IMarketplaceAccess - { - public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) - { - Unavailable(); - } - - public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) - { - Unavailable(); - } - - public void AssertThatBalance(IResolveConstraint constraint, string message = "") - { - Unavailable(); - } - - public decimal GetBalance() - { - Unavailable(); - return 0; - } - - private void Unavailable() - { - Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it."); - throw new InvalidOperationException(); - } - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs deleted file mode 100644 index 8bab9e93..00000000 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ /dev/null @@ -1,24 +0,0 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; -using System.Numerics; -using System.Text; - -namespace CodexDistTestCore.Marketplace -{ - public class MarketplaceController - { - private readonly TestLog log; - private readonly K8sManager k8sManager; - private readonly NumberSource companionGroupNumberSource = new NumberSource(0); - private List companionGroups = new List(); - private GethBootstrapInfo? bootstrapInfo; - - public MarketplaceController(TestLog log, K8sManager k8sManager) - { - this.log = log; - this.k8sManager = k8sManager; - } - - - } -} diff --git a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs b/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs deleted file mode 100644 index 23ad25f7..00000000 --- a/CodexDistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ /dev/null @@ -1,4 +0,0 @@ -namespace CodexDistTestCore.Marketplace -{ - -} diff --git a/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs b/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs deleted file mode 100644 index 0fc0865c..00000000 --- a/CodexDistTestCore/Metrics/K8sPrometheusSpecs.cs +++ /dev/null @@ -1,122 +0,0 @@ -using CodexDistTestCore.Config; -using k8s.Models; - -namespace CodexDistTestCore.Metrics -{ - public class K8sPrometheusSpecs - { - public const string ContainerName = "dtest-prom"; - public const string ConfigFilepath = "/etc/prometheus/prometheus.yml"; - private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; - private const string portName = "prom-1"; - private readonly string config; - - public K8sPrometheusSpecs(int servicePort, int prometheusNumber, string config) - { - ServicePort = servicePort; - PrometheusNumber = prometheusNumber; - this.config = config; - } - - public int ServicePort { get; } - public int PrometheusNumber { get; } - - public string GetDeploymentName() - { - return "test-prom" + PrometheusNumber; - } - - public V1Deployment CreatePrometheusDeployment() - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = GetDeploymentName(), - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = CreateSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = CreateSelector() - }, - Spec = new V1PodSpec - { - Containers = new List - { - new V1Container - { - Name = ContainerName, - Image = dockerImage, - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 9090, - Name = portName - } - }, - Env = new List - { - new V1EnvVar - { - Name = "PROM_CONFIG", - Value = config - } - } - } - } - } - } - } - }; - - return deploymentSpec; - } - - public V1Service CreatePrometheusService() - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = "codex-prom-service" + PrometheusNumber, - NamespaceProperty = K8sCluster.K8sNamespace - }, - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = CreateSelector(), - Ports = new List - { - new V1ServicePort - { - Name = "prom-service" + PrometheusNumber, - Protocol = "TCP", - Port = 9090, - TargetPort = portName, - NodePort = ServicePort - } - } - } - }; - - return serviceSpec; - } - - private Dictionary CreateSelector() - { - return new Dictionary { { "test-prom", "dtest-prom" } }; - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsAccess.cs b/CodexDistTestCore/Metrics/MetricsAccess.cs deleted file mode 100644 index 2f6456e8..00000000 --- a/CodexDistTestCore/Metrics/MetricsAccess.cs +++ /dev/null @@ -1,63 +0,0 @@ -using NUnit.Framework; -using NUnit.Framework.Constraints; - -namespace CodexDistTestCore.Metrics -{ - public interface IMetricsAccess - { - void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); - } - - public class MetricsUnavailable : IMetricsAccess - { - public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") - { - Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); - throw new InvalidOperationException(); - } - } - - public class MetricsAccess : IMetricsAccess - { - private readonly MetricsQuery query; - private readonly OnlineCodexNode node; - - public MetricsAccess(MetricsQuery query, OnlineCodexNode node) - { - this.query = query; - this.node = node; - } - - public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") - { - var metricSet = GetMetricWithTimeout(metricName, node); - var metricValue = metricSet.Values[0].Value; - Assert.That(metricValue, constraint, message); - } - - private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node) - { - var start = DateTime.UtcNow; - - while (true) - { - var mostRecent = GetMostRecent(metricName, node); - if (mostRecent != null) return mostRecent; - if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) - { - Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); - throw new TimeoutException(); - } - - Utils.Sleep(TimeSpan.FromSeconds(2)); - } - } - - private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node) - { - var result = query.GetMostRecent(metricName, node); - if (result == null) return null; - return result.Sets.LastOrDefault(); - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsAggregator.cs b/CodexDistTestCore/Metrics/MetricsAggregator.cs deleted file mode 100644 index adf0c3f6..00000000 --- a/CodexDistTestCore/Metrics/MetricsAggregator.cs +++ /dev/null @@ -1,78 +0,0 @@ -using NUnit.Framework; -using System.Text; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsAggregator - { - private readonly NumberSource prometheusNumberSource = new NumberSource(0); - private readonly TestLog log; - private readonly K8sManager k8sManager; - private readonly Dictionary activePrometheuses = new Dictionary(); - - public MetricsAggregator(TestLog log, K8sManager k8sManager) - { - this.log = log; - this.k8sManager = k8sManager; - } - - public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes) - { - log.Log($"Starting metrics collecting for {nodes.Length} nodes..."); - - var config = GeneratePrometheusConfig(nodes); - var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber()); - var query = new MetricsQuery(prometheus); - activePrometheuses.Add(query, nodes); - - log.Log("Metrics service started."); - - foreach (var node in nodes) - { - node.Metrics = new MetricsAccess(query, node); - } - } - - public void DownloadAllMetrics() - { - var download = new MetricsDownloader(log, activePrometheuses); - download.DownloadAllMetrics(); - } - - private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) - { - var config = ""; - config += "global:\n"; - config += " scrape_interval: 30s\n"; - config += " scrape_timeout: 10s\n"; - config += "\n"; - config += "scrape_configs:\n"; - config += " - job_name: services\n"; - config += " metrics_path: /metrics\n"; - config += " static_configs:\n"; - config += " - targets:\n"; - - foreach (var node in nodes) - { - var ip = node.Group.PodInfo!.Ip; - var port = node.Container.MetricsPort; - config += $" - '{ip}:{port}'\n"; - } - - var bytes = Encoding.ASCII.GetBytes(config); - return Convert.ToBase64String(bytes); - } - } - - public class PrometheusInfo - { - public PrometheusInfo(int servicePort, PodInfo podInfo) - { - ServicePort = servicePort; - PodInfo = podInfo; - } - - public int ServicePort { get; } - public PodInfo PodInfo { get; } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsDownloader.cs b/CodexDistTestCore/Metrics/MetricsDownloader.cs deleted file mode 100644 index 18fd10ba..00000000 --- a/CodexDistTestCore/Metrics/MetricsDownloader.cs +++ /dev/null @@ -1,97 +0,0 @@ -using System.Globalization; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsDownloader - { - private readonly TestLog log; - private readonly Dictionary activePrometheuses; - - public MetricsDownloader(TestLog log, Dictionary activePrometheuses) - { - this.log = log; - this.activePrometheuses = activePrometheuses; - } - - public void DownloadAllMetrics() - { - foreach (var pair in activePrometheuses) - { - DownloadAllMetrics(pair.Key, pair.Value); - } - } - - private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) - { - foreach (var node in nodes) - { - DownloadAllMetricsForNode(query, node); - } - } - - private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) - { - var metrics = query.GetAllMetricsForNode(node); - if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; - - var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); - var map = CreateValueMap(metrics); - - WriteToFile(node.GetName(), headers, map); - } - - private void WriteToFile(string nodeName, string[] headers, Dictionary> map) - { - var file = log.CreateSubfile("csv"); - log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); - - file.WriteRaw(string.Join(",", headers)); - - foreach (var pair in map) - { - file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); - } - } - - private Dictionary> CreateValueMap(Metrics metrics) - { - var map = CreateForAllTimestamps(metrics); - foreach (var metric in metrics.Sets) - { - AddToMap(map, metric); - } - return map; - - } - - private Dictionary> CreateForAllTimestamps(Metrics metrics) - { - var result = new Dictionary>(); - var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); - foreach (var timestamp in timestamps) result.Add(timestamp, new List()); - return result; - } - - private void AddToMap(Dictionary> map, MetricsSet metric) - { - foreach (var key in map.Keys) - { - map[key].Add(GetValueAtTimestamp(key, metric)); - } - } - - private string GetValueAtTimestamp(DateTime key, MetricsSet metric) - { - var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); - if (value == null) return ""; - return value.Value.ToString(CultureInfo.InvariantCulture); - } - - private string FormatTimestamp(DateTime key) - { - var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); - var diff = key - origin; - return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); - } - } -} diff --git a/CodexDistTestCore/Metrics/MetricsQuery.cs b/CodexDistTestCore/Metrics/MetricsQuery.cs deleted file mode 100644 index c028a6c2..00000000 --- a/CodexDistTestCore/Metrics/MetricsQuery.cs +++ /dev/null @@ -1,190 +0,0 @@ -using CodexDistTestCore.Config; -using System.Globalization; - -namespace CodexDistTestCore.Metrics -{ - public class MetricsQuery - { - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly Http http; - - public MetricsQuery(PrometheusInfo prometheusInfo) - { - http = new Http( - k8sCluster.GetIp(), - prometheusInfo.ServicePort, - "api/v1"); - } - - public Metrics? GetMostRecent(string metricName, OnlineCodexNode node) - { - var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); - if (response == null) return null; - - return new Metrics - { - Sets = response.data.result.Select(r => - { - return new MetricsSet - { - Instance = r.metric.instance, - Values = MapSingleValue(r.value) - }; - }).ToArray() - }; - } - - public Metrics? GetMetrics(string metricName) - { - var response = GetAll(metricName); - if (response == null) return null; - return MapResponseToMetrics(response); - } - - public Metrics? GetAllMetricsForNode(OnlineCodexNode node) - { - var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); - if (response.status != "success") return null; - return MapResponseToMetrics(response); - } - - private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) - { - var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); - if (response.status != "success") return null; - return response; - } - - private PrometheusQueryResponse? GetAll(string metricName) - { - var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); - if (response.status != "success") return null; - return response; - } - - private Metrics MapResponseToMetrics(PrometheusQueryResponse response) - { - return new Metrics - { - Sets = response.data.result.Select(r => - { - return new MetricsSet - { - Name = r.metric.__name__, - Instance = r.metric.instance, - Values = MapMultipleValues(r.values) - }; - }).ToArray() - }; - } - - private MetricsSetValue[] MapSingleValue(object[] value) - { - if (value != null && value.Length > 0) - { - return new[] - { - MapValue(value) - }; - } - return Array.Empty(); - } - - private MetricsSetValue[] MapMultipleValues(object[][] values) - { - if (values != null && values.Length > 0) - { - return values.Select(v => MapValue(v)).ToArray(); - } - return Array.Empty(); - } - - private MetricsSetValue MapValue(object[] value) - { - if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); - - return new MetricsSetValue - { - Timestamp = ToTimestamp(value[0]), - Value = ToValue(value[1]) - }; - } - - private string GetInstanceNameForNode(OnlineCodexNode node) - { - var pod = node.Group.PodInfo!; - return $"{pod.Ip}:{node.Container.MetricsPort}"; - } - - private string GetInstanceStringForNode(OnlineCodexNode node) - { - return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; - } - - private string GetQueryTimeRange() - { - return "[12h]"; - } - - private double ToValue(object v) - { - return Convert.ToDouble(v, CultureInfo.InvariantCulture); - } - - private DateTime ToTimestamp(object v) - { - var unixSeconds = ToValue(v); - return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); - } - } - - public class Metrics - { - public MetricsSet[] Sets { get; set; } = Array.Empty(); - } - - public class MetricsSet - { - public string Name { get; set; } = string.Empty; - public string Instance { get; set; } = string.Empty; - public MetricsSetValue[] Values { get; set; } = Array.Empty(); - } - - public class MetricsSetValue - { - public DateTime Timestamp { get; set; } - public double Value { get; set; } - } - - public class PrometheusQueryResponse - { - public string status { get; set; } = string.Empty; - public PrometheusQueryResponseData data { get; set; } = new(); - } - - public class PrometheusQueryResponseData - { - public string resultType { get; set; } = string.Empty; - public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); - } - - public class PrometheusQueryResponseDataResultEntry - { - public ResultEntryMetric metric { get; set; } = new(); - public object[] value { get; set; } = Array.Empty(); - public object[][] values { get; set; } = Array.Empty(); - } - - public class ResultEntryMetric - { - public string __name__ { get; set; } = string.Empty; - public string instance { get; set; } = string.Empty; - public string job { get; set; } = string.Empty; - } - - public class PrometheusAllNamesResponse - { - public string status { get; set; } = string.Empty; - public string[] data { get; set; } = Array.Empty(); - } -} diff --git a/CodexDistTestCore/NumberSource.cs b/CodexDistTestCore/NumberSource.cs deleted file mode 100644 index 43386103..00000000 --- a/CodexDistTestCore/NumberSource.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace CodexDistTestCore -{ - public class NumberSource - { - private int number; - - public NumberSource(int start) - { - number = start; - } - - public int GetNextNumber() - { - var n = number; - number++; - return n; - } - } -} diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs deleted file mode 100644 index 84003a61..00000000 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ /dev/null @@ -1,97 +0,0 @@ -using CodexDistTestCore.Marketplace; - -namespace CodexDistTestCore -{ - public interface IOfflineCodexNodes - { - IOfflineCodexNodes At(Location location); - IOfflineCodexNodes WithLogLevel(CodexLogLevel level); - IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); - IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); - IOfflineCodexNodes EnableMetrics(); - IOfflineCodexNodes EnableMarketplace(int initialBalance); - ICodexNodeGroup BringOnline(); - } - - public enum Location - { - Unspecified, - BensLaptop, - BensOldGamingMachine, - } - - public class OfflineCodexNodes : IOfflineCodexNodes - { - private readonly IK8sManager k8SManager; - - public int NumberOfNodes { get; } - public Location Location { get; private set; } - public CodexLogLevel? LogLevel { get; private set; } - public IOnlineCodexNode? BootstrapNode { get; private set; } - public ByteSize? StorageQuota { get; private set; } - public bool MetricsEnabled { get; private set; } - public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } - - public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) - { - this.k8SManager = k8SManager; - NumberOfNodes = numberOfNodes; - Location = Location.Unspecified; - MetricsEnabled = false; - } - - public ICodexNodeGroup BringOnline() - { - return k8SManager.BringOnline(this); - } - - public IOfflineCodexNodes At(Location location) - { - Location = location; - return this; - } - - public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node) - { - BootstrapNode = node; - return this; - } - - public IOfflineCodexNodes WithLogLevel(CodexLogLevel level) - { - LogLevel = level; - return this; - } - - public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota) - { - StorageQuota = storageQuota; - return this; - } - - public IOfflineCodexNodes EnableMetrics() - { - MetricsEnabled = true; - return this; - } - - public IOfflineCodexNodes EnableMarketplace(int initialBalance) - { - MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); - return this; - } - - public string Describe() - { - var args = string.Join(',', DescribeArgs()); - return $"{NumberOfNodes} CodexNodes with [{args}]"; - } - - private IEnumerable DescribeArgs() - { - if (LogLevel != null) yield return ($"LogLevel={LogLevel}"); - if (BootstrapNode != null) yield return ("BootstrapNode=set-not-shown-here"); - if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}"); - } - } -} diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs deleted file mode 100644 index 64f9c6d8..00000000 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ /dev/null @@ -1,141 +0,0 @@ -using CodexDistTestCore.Config; -using CodexDistTestCore.Marketplace; -using CodexDistTestCore.Metrics; -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IOnlineCodexNode - { - CodexDebugResponse GetDebugInfo(); - ContentId UploadFile(TestFile file); - TestFile? DownloadContent(ContentId contentId); - void ConnectToPeer(IOnlineCodexNode node); - ICodexNodeLog DownloadLog(); - IMetricsAccess Metrics { get; } - IMarketplaceAccess Marketplace { get; } - } - - public class OnlineCodexNode : IOnlineCodexNode - { - private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; - private const string UploadFailedMessage = "Unable to store block"; - - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly TestLog log; - private readonly IFileManager fileManager; - - public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container) - { - this.log = log; - this.fileManager = fileManager; - Container = container; - } - - public CodexNodeContainer Container { get; } - public CodexNodeGroup Group { get; internal set; } = null!; - public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable(); - public IMarketplaceAccess Marketplace { set; get; } = new MarketplaceUnavailable(); - - public string GetName() - { - return $"<{Container.Name}>"; - } - - public CodexDebugResponse GetDebugInfo() - { - var response = Http().HttpGetJson("debug/info"); - Log($"Got DebugInfo with id: '{response.id}'."); - return response; - } - - public ContentId UploadFile(TestFile file) - { - Log($"Uploading file of size {file.GetFileSize()}..."); - using var fileStream = File.OpenRead(file.Filename); - var response = Http().HttpPostStream("upload", fileStream); - if (response.StartsWith(UploadFailedMessage)) - { - Assert.Fail("Node failed to store block."); - } - Log($"Uploaded file. Received contentId: '{response}'."); - return new ContentId(response); - } - - public TestFile? DownloadContent(ContentId contentId) - { - Log($"Downloading for contentId: '{contentId.Id}'..."); - var file = fileManager.CreateEmptyTestFile(); - DownloadToFile(contentId.Id, file); - Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'."); - return file; - } - - public void ConnectToPeer(IOnlineCodexNode node) - { - var peer = (OnlineCodexNode)node; - - Log($"Connecting to peer {peer.GetName()}..."); - var peerInfo = node.GetDebugInfo(); - var peerId = peerInfo.id; - var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo); - - var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); - - Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); - Log($"Successfully connected to peer {peer.GetName()}."); - } - - public ICodexNodeLog DownloadLog() - { - return Group.DownloadLog(this); - } - - public string Describe() - { - return $"{Group.Describe()} contains {GetName()}"; - } - - private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) - { - var multiAddress = peerInfo.addrs.First(); - // Todo: Is there a case where First address in list is not the way? - - if (Group == peer.Group) - { - return multiAddress; - } - - // The peer we want to connect is in a different pod. - // We must replace the default IP with the pod IP in the multiAddress. - return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip); - } - - private void DownloadToFile(string contentId, TestFile file) - { - using var fileStream = File.OpenWrite(file.Filename); - using var downloadStream = Http().HttpGetStream("download/" + contentId); - downloadStream.CopyTo(fileStream); - } - - private Http Http() - { - return new Http(ip: k8sCluster.GetIp(), port: Container.ServicePort, baseUrl: "/api/codex/v1"); - } - - private void Log(string msg) - { - log.Log($"{GetName()}: {msg}"); - } - } - - public class ContentId - { - public ContentId(string id) - { - Id = id; - } - - public string Id { get; } - } -} diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs deleted file mode 100644 index e09a57ce..00000000 --- a/CodexDistTestCore/PodLogDownloader.cs +++ /dev/null @@ -1,64 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - public interface IPodLogHandler - { - void Log(Stream log); - } - - public class PodLogDownloader - { - public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure"; - - private readonly TestLog log; - private readonly IK8sManager k8SManager; - - public PodLogDownloader(TestLog log, IK8sManager k8sManager) - { - this.log = log; - k8SManager = k8sManager; - } - - public CodexNodeLog DownloadLog(OnlineCodexNode node) - { - var description = node.Describe(); - var subFile = log.CreateSubfile(); - - log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); - var handler = new PodLogDownloadHandler(description, subFile); - k8SManager.FetchPodLog(node, handler); - return handler.CreateCodexNodeLog(); - } - } - - public class PodLogDownloadHandler : IPodLogHandler - { - private readonly string description; - private readonly LogFile log; - - public PodLogDownloadHandler(string description, LogFile log) - { - this.description = description; - this.log = log; - } - - public CodexNodeLog CreateCodexNodeLog() - { - return new CodexNodeLog(log); - } - - public void Log(Stream stream) - { - log.Write($"{description} -->> {log.FilenameWithoutPath}"); - log.WriteRaw(description); - var reader = new StreamReader(stream); - var line = reader.ReadLine(); - while (line != null) - { - log.WriteRaw(line); - line = reader.ReadLine(); - } - } - } -} diff --git a/CodexDistTestCore/Timing.cs b/CodexDistTestCore/Timing.cs deleted file mode 100644 index cfa94568..00000000 --- a/CodexDistTestCore/Timing.cs +++ /dev/null @@ -1,131 +0,0 @@ -using NUnit.Framework; - -namespace CodexDistTestCore -{ - [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class UseLongTimeoutsAttribute : PropertyAttribute - { - public UseLongTimeoutsAttribute() - : base(Timing.UseLongTimeoutsKey) - { - } - } - - public static class Timing - { - public const string UseLongTimeoutsKey = "UseLongTimeouts"; - - public static TimeSpan HttpCallTimeout() - { - return GetTimes().HttpCallTimeout(); - } - - public static int HttpCallRetryCount() - { - return GetTimes().HttpCallRetryCount(); - } - - public static void HttpCallRetryDelay() - { - Utils.Sleep(GetTimes().HttpCallRetryDelay()); - } - - public static void WaitForK8sServiceDelay() - { - Utils.Sleep(GetTimes().WaitForK8sServiceDelay()); - } - - public static TimeSpan K8sOperationTimeout() - { - return GetTimes().K8sOperationTimeout(); - } - - public static TimeSpan WaitForMetricTimeout() - { - return GetTimes().WaitForMetricTimeout(); - } - - private static ITimeSet GetTimes() - { - var testProperties = TestContext.CurrentContext.Test.Properties; - if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet(); - return new DefaultTimeSet(); - } - } - - public interface ITimeSet - { - TimeSpan HttpCallTimeout(); - int HttpCallRetryCount(); - TimeSpan HttpCallRetryDelay(); - TimeSpan WaitForK8sServiceDelay(); - TimeSpan K8sOperationTimeout(); - TimeSpan WaitForMetricTimeout(); - } - - public class DefaultTimeSet : ITimeSet - { - public TimeSpan HttpCallTimeout() - { - return TimeSpan.FromSeconds(10); - } - - public int HttpCallRetryCount() - { - return 5; - } - - public TimeSpan HttpCallRetryDelay() - { - return TimeSpan.FromSeconds(3); - } - - public TimeSpan WaitForK8sServiceDelay() - { - return TimeSpan.FromSeconds(1); - } - - public TimeSpan K8sOperationTimeout() - { - return TimeSpan.FromMinutes(5); - } - - public TimeSpan WaitForMetricTimeout() - { - return TimeSpan.FromSeconds(30); - } - } - - public class LongTimeSet : ITimeSet - { - public TimeSpan HttpCallTimeout() - { - return TimeSpan.FromHours(2); - } - - public int HttpCallRetryCount() - { - return 2; - } - - public TimeSpan HttpCallRetryDelay() - { - return TimeSpan.FromMinutes(5); - } - - public TimeSpan WaitForK8sServiceDelay() - { - return TimeSpan.FromSeconds(10); - } - - public TimeSpan K8sOperationTimeout() - { - return TimeSpan.FromMinutes(15); - } - - public TimeSpan WaitForMetricTimeout() - { - return TimeSpan.FromMinutes(5); - } - } -} diff --git a/CodexDistTestCore/TryContract.cs b/CodexDistTestCore/TryContract.cs deleted file mode 100644 index efc9d5a0..00000000 --- a/CodexDistTestCore/TryContract.cs +++ /dev/null @@ -1,101 +0,0 @@ -using Nethereum.Web3; -using Nethereum.ABI.FunctionEncoding.Attributes; -using Nethereum.Contracts.CQS; -using Nethereum.Util; -using Nethereum.Web3.Accounts; -using Nethereum.Hex.HexConvertors.Extensions; -using Nethereum.Contracts; -using Nethereum.Contracts.Extensions; -using System.Numerics; -using NUnit.Framework; - -// https://docs.nethereum.com/en/latest/nethereum-smartcontrats-gettingstarted/ - -namespace CodexDistTestCore -{ - public class TryContract - { - [Test] - [Ignore("aaa")] - public void DoThing() - { - var url = "http://testchain.nethereum.com:8545"; - var privateKey = "0x7580e7fb49df1c861f0050fae31c2224c6aba908e116b8da44ee8cd927b990b0"; - var account = new Account(privateKey); - var web3 = new Web3(account, url); - - // Deploy contract: - var deploymentMessage = new StandardTokenDeployment - { - TotalSupply = 100000 - }; - var deploymentHandler = web3.Eth.GetContractDeploymentHandler(); - var transactionReceipt = Utils.Wait(deploymentHandler.SendRequestAndWaitForReceiptAsync(deploymentMessage)); - var contractAddress = transactionReceipt.ContractAddress; - - // Get balance: - var balanceOfFunctionMessage = new BalanceOfFunction() - { - Owner = account.Address, - }; - - var balanceHandler = web3.Eth.GetContractQueryHandler(); - var balance = Utils.Wait(balanceHandler.QueryAsync(contractAddress, balanceOfFunctionMessage)); - long asInt = ((long)balance); - - // Transfer: - var receiverAddress = "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"; - var transferHandler = web3.Eth.GetContractTransactionHandler(); - var transfer = new TransferFunction() - { - To = receiverAddress, - TokenAmount = 100 - }; - var transferReceipt = Utils.Wait(transferHandler.SendRequestAndWaitForReceiptAsync(contractAddress, transfer)); - - // Signing: - var signedTransaction = Utils.Wait(transferHandler.SignTransactionAsync(contractAddress, transfer)); - } - } - - public class StandardTokenDeployment : ContractDeploymentMessage - { - - public static string BYTECODE = "0x60606040526040516020806106f5833981016040528080519060200190919050505b80600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005081905550806000600050819055505b506106868061006f6000396000f360606040523615610074576000357c010000000000000000000000000000000000000000000000000000000090048063095ea7b31461008157806318160ddd146100b657806323b872dd146100d957806370a0823114610117578063a9059cbb14610143578063dd62ed3e1461017857610074565b61007f5b610002565b565b005b6100a060048080359060200190919080359060200190919050506101ad565b6040518082815260200191505060405180910390f35b6100c36004805050610674565b6040518082815260200191505060405180910390f35b6101016004808035906020019091908035906020019091908035906020019091905050610281565b6040518082815260200191505060405180910390f35b61012d600480803590602001909190505061048d565b6040518082815260200191505060405180910390f35b61016260048080359060200190919080359060200190919050506104cb565b6040518082815260200191505060405180910390f35b610197600480803590602001909190803590602001909190505061060b565b6040518082815260200191505060405180910390f35b600081600260005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008573ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925846040518082815260200191505060405180910390a36001905061027b565b92915050565b600081600160005060008673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561031b575081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505410155b80156103275750600082115b1561047c5781600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff168473ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a381600160005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600260005060008673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060003373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505403925050819055506001905061048656610485565b60009050610486565b5b9392505050565b6000600160005060008373ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000505490506104c6565b919050565b600081600160005060003373ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600050541015801561050c5750600082115b156105fb5781600160005060003373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282825054039250508190555081600160005060008573ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828282505401925050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a36001905061060556610604565b60009050610605565b5b92915050565b6000600260005060008473ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005060008373ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060005054905061066e565b92915050565b60006000600050549050610683565b9056"; - - public StandardTokenDeployment() : base(BYTECODE) { } - - [Parameter("uint256", "totalSupply")] - public BigInteger TotalSupply { get; set; } - } - - [Function("balanceOf", "uint256")] - public class BalanceOfFunction : FunctionMessage - { - [Parameter("address", "_owner", 1)] - public string Owner { get; set; } - } - - [Function("transfer", "bool")] - public class TransferFunction : FunctionMessage - { - [Parameter("address", "_to", 1)] - public string To { get; set; } - - [Parameter("uint256", "_value", 2)] - public BigInteger TokenAmount { get; set; } - } - - [Event("Transfer")] - public class TransferEventDTO : IEventDTO - { - [Parameter("address", "_from", 1, true)] - public string From { get; set; } - - [Parameter("address", "_to", 2, true)] - public string To { get; set; } - - [Parameter("uint256", "_value", 3, false)] - public BigInteger Value { get; set; } - } -} diff --git a/CodexDistTestCore/Utils.cs b/CodexDistTestCore/Utils.cs deleted file mode 100644 index ae3e761e..00000000 --- a/CodexDistTestCore/Utils.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace CodexDistTestCore -{ - public static class Utils - { - - } -} diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index a7f0c9ba..3db5d596 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -1,4 +1,5 @@ -using CodexDistTestCore; +using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index c39069c6..7acd78e9 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -1,4 +1,5 @@ -using CodexDistTestCore; +using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests diff --git a/LongTests/TestsLong.csproj b/LongTests/TestsLong.csproj index fc5152fb..136951d8 100644 --- a/LongTests/TestsLong.csproj +++ b/LongTests/TestsLong.csproj @@ -13,7 +13,7 @@ - + diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 1f5c04fc..4bc64fc6 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -7,8 +7,6 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestsLong", "LongTests\TestsLong.csproj", "{AFCE270E-F844-4A7C-9006-69AE622BB1F4}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDistTestCore", "CodexDistTestCore\CodexDistTestCore.csproj", "{19306DE1-CEE5-4F7B-AA5D-FD91926D853D}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "DistTestCore\DistTestCore.csproj", "{47F31305-6E68-4827-8E39-7B41DAA1CE7A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "KubernetesWorkflow", "KubernetesWorkflow\KubernetesWorkflow.csproj", "{359123AA-3D9B-4442-80F4-19E32E3EC9EA}" @@ -17,7 +15,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Utils\Utils.csproj EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging.csproj", "{8481A4A6-4BDD-41B0-A3EB-EF53F7BD40D1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -33,10 +31,6 @@ Global {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Debug|Any CPU.Build.0 = Debug|Any CPU {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Release|Any CPU.ActiveCfg = Release|Any CPU {AFCE270E-F844-4A7C-9006-69AE622BB1F4}.Release|Any CPU.Build.0 = Release|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {19306DE1-CEE5-4F7B-AA5D-FD91926D853D}.Release|Any CPU.Build.0 = Release|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Debug|Any CPU.Build.0 = Debug|Any CPU {47F31305-6E68-4827-8E39-7B41DAA1CE7A}.Release|Any CPU.ActiveCfg = Release|Any CPU