From d23f5aa29de4316d04fe273419061d7af784bef3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 21 Jun 2023 08:28:40 +0200 Subject: [PATCH 01/23] Initial setup for continuous test runner --- ContinuousTests/Configuration.cs | 78 ++++++++ ContinuousTests/ContinuousTestRunner.cs | 111 +++++++++++ ContinuousTests/ContinuousTests.csproj | 19 ++ ContinuousTests/IContinuousTest.cs | 33 ++++ ContinuousTests/Program.cs | 10 + ContinuousTests/TestFinder.cs | 24 +++ ContinuousTests/TestRun.cs | 87 +++++++++ DistTestCore/Codex/CodexAccess.cs | 160 +--------------- DistTestCore/Codex/CodexNode.cs | 173 ++++++++++++++++++ DistTestCore/Configuration.cs | 3 +- DistTestCore/FileManager.cs | 8 +- DistTestCore/Http.cs | 7 +- DistTestCore/Marketplace/MarketplaceAccess.cs | 4 +- DistTestCore/OnlineCodexNode.cs | 12 +- KubernetesWorkflow/RunningContainers.cs | 22 +-- KubernetesWorkflow/StartupWorkflow.cs | 9 +- Utils/Address.cs | 14 ++ cs-codex-dist-testing.sln | 6 + 18 files changed, 588 insertions(+), 192 deletions(-) create mode 100644 ContinuousTests/Configuration.cs create mode 100644 ContinuousTests/ContinuousTestRunner.cs create mode 100644 ContinuousTests/ContinuousTests.csproj create mode 100644 ContinuousTests/IContinuousTest.cs create mode 100644 ContinuousTests/Program.cs create mode 100644 ContinuousTests/TestFinder.cs create mode 100644 ContinuousTests/TestRun.cs create mode 100644 DistTestCore/Codex/CodexNode.cs create mode 100644 Utils/Address.cs diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs new file mode 100644 index 0000000..5ccb845 --- /dev/null +++ b/ContinuousTests/Configuration.cs @@ -0,0 +1,78 @@ +using Newtonsoft.Json; + +namespace ContinuousTests +{ + public class Configuration + { + public string LogPath { get; set; } = string.Empty; + public string[] CodexUrls { get; set; } = Array.Empty(); + public int SleepSecondsPerTest { get; set; } + } + + public class ConfigLoader + { + private const string filename = "config.json"; + + public Configuration Load() + { + var config = Read(); + Validate(config); + return config; + } + + private Configuration Read() + { + if (File.Exists(filename)) + { + var lines = File.ReadAllText(filename); + try + { + var result = JsonConvert.DeserializeObject(lines); + if (result != null) return result; + } + catch { } + } + + var logPath = Environment.GetEnvironmentVariable("LOGPATH"); + var codexUrls = Environment.GetEnvironmentVariable("CODEXURLS"); + var sleep = Environment.GetEnvironmentVariable("SLEEPSECONDSPERTEST"); + + if (!string.IsNullOrEmpty(logPath) && !string.IsNullOrEmpty(codexUrls) && !string.IsNullOrEmpty(sleep)) + { + var urls = codexUrls.Split(';', StringSplitOptions.RemoveEmptyEntries); + var ms = 0; + if (int.TryParse(sleep, out ms)) + { + if (urls.Length > 0) + { + return new Configuration { LogPath = logPath, CodexUrls = urls, SleepSecondsPerTest = ms }; + } + } + } + + throw new Exception($"Unable to load configuration from '{filename}', and " + + $"unable to load configuration from environment variables 'LOGPATH' and 'CODEXURLS', and 'SLEEPSECONDSPERTEST'. " + + $"(semi-colon-separated URLs) " + + $"Create the configuration file or set the environment veriables."); + } + + private void Validate(Configuration configuration) + { + if (configuration.SleepSecondsPerTest < 10) + { + Console.WriteLine("Warning: configuration.SleepMsPerTest was less than 10 seconds. Using 10 seconds instead!"); + configuration.SleepSecondsPerTest = 10; + } + + if (string.IsNullOrEmpty(configuration.LogPath)) + { + throw new Exception($"Unvalid logpath set: '{configuration.LogPath}'"); + } + + if (!configuration.CodexUrls.Any()) + { + throw new Exception("No Codex URLs found."); + } + } + } +} diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs new file mode 100644 index 0000000..caa9fbf --- /dev/null +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -0,0 +1,111 @@ +using DistTestCore; +using DistTestCore.Codex; +using Logging; +using Utils; + +namespace ContinuousTests +{ + public class ContinuousTestRunner + { + private readonly ConfigLoader configLoader = new ConfigLoader(); + private readonly TestFinder testFinder = new TestFinder(); + + public void Run() + { + var config = configLoader.Load(); + var log = new TestLog(config.LogPath, true); + + log.Log("Starting continuous test run..."); + log.Log("Checking configuration..."); + PreflightCheck(config); + log.Log("Contacting Codex nodes..."); + var nodes = CreateCodexNodes(log, new LongTimeSet(), config); + log.Log("OK"); + log.Log(""); + + while (true) + { + var run = new TestRun(config, log, testFinder, nodes); + + try + { + run.Run(); + } + catch (Exception ex) + { + log.Error($"Exception during test run: " + ex); + } + + Thread.Sleep(config.SleepSecondsPerTest * 1000); + } + } + + private void PreflightCheck(Configuration config) + { + var tests = testFinder.GetTests(); + if (!tests.Any()) + { + throw new Exception("Unable to find any tests."); + } + + var errors = new List(); + foreach (var test in tests) + { + if (test.RequiredNumberOfNodes > config.CodexUrls.Length) + { + errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Configuration only has {config.CodexUrls.Length}"); + } + } + + if (errors.Any()) + { + throw new Exception("Prerun check failed: " + string.Join(", ", errors)); + } + } + + private CodexNode[] CreateCodexNodes(BaseLog log, ITimeSet timeSet, Configuration config) + { + var nodes = config.CodexUrls.Select(url => + { + var address = new Address(url, 1234); + return new CodexNode(log, timeSet, address); + }).ToArray(); + + var pass = true; + foreach (var n in nodes) + { + log.Log($"Checking '{n.Address.Host}'..."); + + if (EnsureOnline(n)) + { + log.Log("OK"); + } + else + { + log.Error($"No response from '{n.Address.Host}'."); + pass = false; + } + } + if (!pass) + { + throw new Exception("Not all codex nodes responded."); + } + + return nodes; + } + + private bool EnsureOnline(CodexNode n) + { + try + { + var info = n.GetDebugInfo(); + if (info == null || string.IsNullOrEmpty(info.id)) return false; + } + catch + { + return false; + } + return true; + } + } +} diff --git a/ContinuousTests/ContinuousTests.csproj b/ContinuousTests/ContinuousTests.csproj new file mode 100644 index 0000000..593bf86 --- /dev/null +++ b/ContinuousTests/ContinuousTests.csproj @@ -0,0 +1,19 @@ + + + + Exe + net7.0 + enable + enable + + + + + + + + + + + + diff --git a/ContinuousTests/IContinuousTest.cs b/ContinuousTests/IContinuousTest.cs new file mode 100644 index 0000000..0374ffd --- /dev/null +++ b/ContinuousTests/IContinuousTest.cs @@ -0,0 +1,33 @@ +using DistTestCore; +using DistTestCore.Codex; +using Logging; + +namespace ContinuousTests +{ + public interface IContinuousTest + { + string Name { get; } + int RequiredNumberOfNodes { get; } + + void Run(); + } + + public abstract class ContinuousTest : IContinuousTest + { + public CodexNode[] Nodes { get; set; } = null!; + public BaseLog Log { get; set; } = null!; + public FileManager FileManager { get; set; } = null!; + + public abstract int RequiredNumberOfNodes { get; } + + public string Name + { + get + { + return GetType().Name; + } + } + + public abstract void Run(); + } +} diff --git a/ContinuousTests/Program.cs b/ContinuousTests/Program.cs new file mode 100644 index 0000000..d3fcb1d --- /dev/null +++ b/ContinuousTests/Program.cs @@ -0,0 +1,10 @@ +using ContinuousTests; + +public class Program +{ + public static void Main(string[] args) + { + var runner = new ContinuousTestRunner(); + runner.Run(); + } +} diff --git a/ContinuousTests/TestFinder.cs b/ContinuousTests/TestFinder.cs new file mode 100644 index 0000000..349e957 --- /dev/null +++ b/ContinuousTests/TestFinder.cs @@ -0,0 +1,24 @@ +namespace ContinuousTests +{ + public class TestFinder + { + private readonly List testList = new List(); + + public IContinuousTest[] GetTests() + { + if (!testList.Any()) FindTests(); + return testList.ToArray(); + } + + private void FindTests() + { + var types = GetType().Assembly.GetTypes(); + var testTypes = types.Where(t => typeof(IContinuousTest).IsAssignableFrom(t) && !t.IsAbstract); + foreach (var testType in testTypes) + { + var t = Activator.CreateInstance(testType); + testList.Add((IContinuousTest)t!); + } + } + } +} diff --git a/ContinuousTests/TestRun.cs b/ContinuousTests/TestRun.cs new file mode 100644 index 0000000..9138744 --- /dev/null +++ b/ContinuousTests/TestRun.cs @@ -0,0 +1,87 @@ +using DistTestCore; +using DistTestCore.Codex; +using Logging; + +namespace ContinuousTests +{ + public class TestRun + { + private readonly Random random = new Random(); + private readonly Configuration config; + private readonly BaseLog log; + private readonly TestFinder testFinder; + private readonly CodexNode[] nodes; + private readonly FileManager fileManager; + + public TestRun(Configuration config, BaseLog log, TestFinder testFinder, CodexNode[] nodes) + { + this.config = config; + this.log = log; + this.testFinder = testFinder; + this.nodes = nodes; + fileManager = new FileManager(log, new DistTestCore.Configuration()); + } + + public void Run() + { + var remainingTests = testFinder.GetTests().ToList(); + while (remainingTests.Any()) + { + var test = PickOneRandom(remainingTests); + var selectedNodes = SelectRandomNodes(test.RequiredNumberOfNodes); + AssignEssentials(test, selectedNodes); + fileManager.PushFileSet(); + + log.Log($"Start '{test.Name}'"); + try + { + test.Run(); + log.Log($"'{test.Name}' = Passed"); + } + catch + { + log.Log($"'{test.Name}' = Failed"); + } + + fileManager.PopFileSet(); + ClearEssentials(test); + Thread.Sleep(config.SleepSecondsPerTest * 1000); + } + } + + private void AssignEssentials(IContinuousTest test, CodexNode[] nodes) + { + var t = (ContinuousTest)test; + t.Nodes = nodes; + t.Log = log; + t.FileManager = fileManager; + } + + private void ClearEssentials(IContinuousTest test) + { + var t = (ContinuousTest)test; + t.Nodes = null!; + t.Log = null!; + t.FileManager = null!; + } + + private CodexNode[] SelectRandomNodes(int number) + { + var remainingNodes = nodes.ToList(); + var result = new CodexNode[number]; + for (var i = 0; i < number; i++) + { + result[i] = PickOneRandom(remainingNodes); + } + return result; + } + + private T PickOneRandom(List remainingItems) + { + var i = random.Next(0, remainingItems.Count); + var result = remainingItems[i]; + remainingItems.RemoveAt(i); + return result; + } + } +} diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 4d6f1da..ee0e2a5 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -10,68 +10,19 @@ namespace DistTestCore.Codex { this.lifecycle = lifecycle; Container = runningContainer; + + var address = lifecycle.Configuration.GetAddress(Container); + Node = new CodexNode(lifecycle.Log, lifecycle.TimeSet, address); } public RunningContainer Container { get; } - - public CodexDebugResponse GetDebugInfo() - { - return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); - } - - public CodexDebugPeerResponse GetDebugPeer(string peerId) - { - return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); - } - - public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) - { - var http = Http(timeout); - var str = http.HttpGetString($"debug/peer/{peerId}"); - - if (str.ToLowerInvariant() == "unable to find peer!") - { - return new CodexDebugPeerResponse - { - IsPeerFound = false - }; - } - - var result = http.TryJsonDeserialize(str); - result.IsPeerFound = true; - return result; - } - - public string UploadFile(FileStream fileStream) - { - return Http().HttpPostStream("upload", fileStream); - } - - public Stream DownloadFile(string contentId) - { - return Http().HttpGetStream("download/" + contentId); - } - - public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request) - { - return Http().HttpPostJson("sales/availability", request); - } - - public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId) - { - return Http().HttpPostJson($"storage/request/{contentId}", request); - } - - public string ConnectToPeer(string peerId, string peerMultiAddress) - { - return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); - } + public CodexNode Node { get; } public void EnsureOnline() { try { - var debugInfo = GetDebugInfo(); + var debugInfo = Node.GetDebugInfo(); if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup."); var nodePeerId = debugInfo.id; @@ -85,106 +36,5 @@ namespace DistTestCore.Codex throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e); } } - - private Http Http(TimeSpan? timeoutOverride = null) - { - var address = lifecycle.Configuration.GetAddress(Container); - return new Http(lifecycle.Log, lifecycle.TimeSet, address, baseUrl: "/api/codex/v1", timeoutOverride); - } - } - - public class CodexDebugResponse - { - public string id { get; set; } = string.Empty; - public string[] addrs { get; set; } = new string[0]; - public string repo { get; set; } = string.Empty; - public string spr { get; set; } = string.Empty; - public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty(); - public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty(); - public CodexDebugVersionResponse codex { get; set; } = new(); - public CodexDebugTableResponse table { get; set; } = new(); - } - - public class CodexDebugTableResponse - { - public CodexDebugTableNodeResponse localNode { get; set; } = new(); - public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty(); - } - - public class CodexDebugTableNodeResponse - { - public string nodeId { get; set; } = string.Empty; - public string peerId { get; set; } = string.Empty; - public string record { get; set; } = string.Empty; - public string address { get; set; } = string.Empty; - public bool seen { get; set; } - } - - public class EnginePeerResponse - { - public string peerId { get; set; } = string.Empty; - public EnginePeerContextResponse context { get; set; } = new(); - } - - public class EnginePeerContextResponse - { - public int blocks { get; set; } = 0; - public int peerWants { get; set; } = 0; - public int exchanged { get; set; } = 0; - public string lastExchange { get; set; } = string.Empty; - } - - public class SwitchPeerResponse - { - public string peerId { get; set; } = string.Empty; - public string key { get; set; } = string.Empty; - } - - public class CodexDebugVersionResponse - { - public string version { get; set; } = string.Empty; - public string revision { get; set; } = string.Empty; - } - - public class CodexDebugPeerResponse - { - public bool IsPeerFound { get; set; } - - public string peerId { get; set; } = string.Empty; - public long seqNo { get; set; } - public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty(); - } - - public class CodexDebugPeerAddressResponse - { - public string address { get; set; } = string.Empty; - } - - public class CodexSalesAvailabilityRequest - { - public string size { get; set; } = string.Empty; - public string duration { get; set; } = string.Empty; - public string minPrice { get; set; } = string.Empty; - public string maxCollateral { get; set; } = string.Empty; - } - - public class CodexSalesAvailabilityResponse - { - public string id { get; set; } = string.Empty; - public string size { get; set; } = string.Empty; - public string duration { get; set; } = string.Empty; - public string minPrice { get; set; } = string.Empty; - public string maxCollateral { get; set; } = string.Empty; - } - - public class CodexSalesRequestStorageRequest - { - public string duration { get; set; } = string.Empty; - public string proofProbability { get; set; } = string.Empty; - public string reward { get; set; } = string.Empty; - public string collateral { get; set; } = string.Empty; - public string? expiry { get; set; } - public uint? nodes { get; set; } - public uint? tolerance { get; set;} } } diff --git a/DistTestCore/Codex/CodexNode.cs b/DistTestCore/Codex/CodexNode.cs new file mode 100644 index 0000000..510c2fb --- /dev/null +++ b/DistTestCore/Codex/CodexNode.cs @@ -0,0 +1,173 @@ +using Logging; +using Utils; + +namespace DistTestCore.Codex +{ + public class CodexNode + { + private readonly BaseLog log; + private readonly ITimeSet timeSet; + + public CodexNode(BaseLog log, ITimeSet timeSet, Address address) + { + this.log = log; + this.timeSet = timeSet; + Address = address; + } + + public Address Address { get; } + + public CodexDebugResponse GetDebugInfo() + { + return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId) + { + return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) + { + var http = Http(timeout); + var str = http.HttpGetString($"debug/peer/{peerId}"); + + if (str.ToLowerInvariant() == "unable to find peer!") + { + return new CodexDebugPeerResponse + { + IsPeerFound = false + }; + } + + var result = http.TryJsonDeserialize(str); + result.IsPeerFound = true; + return result; + } + + public string UploadFile(FileStream fileStream) + { + return Http().HttpPostStream("upload", fileStream); + } + + public Stream DownloadFile(string contentId) + { + return Http().HttpGetStream("download/" + contentId); + } + + public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request) + { + return Http().HttpPostJson("sales/availability", request); + } + + public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId) + { + return Http().HttpPostJson($"storage/request/{contentId}", request); + } + + public string ConnectToPeer(string peerId, string peerMultiAddress) + { + return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + } + + private Http Http(TimeSpan? timeoutOverride = null) + { + return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride); + } + } + + public class CodexDebugResponse + { + public string id { get; set; } = string.Empty; + public string[] addrs { get; set; } = new string[0]; + public string repo { get; set; } = string.Empty; + public string spr { get; set; } = string.Empty; + public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty(); + public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty(); + public CodexDebugVersionResponse codex { get; set; } = new(); + public CodexDebugTableResponse table { get; set; } = new(); + } + + public class CodexDebugTableResponse + { + public CodexDebugTableNodeResponse localNode { get; set; } = new(); + public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty(); + } + + public class CodexDebugTableNodeResponse + { + public string nodeId { get; set; } = string.Empty; + public string peerId { get; set; } = string.Empty; + public string record { get; set; } = string.Empty; + public string address { get; set; } = string.Empty; + public bool seen { get; set; } + } + + public class EnginePeerResponse + { + public string peerId { get; set; } = string.Empty; + public EnginePeerContextResponse context { get; set; } = new(); + } + + public class EnginePeerContextResponse + { + public int blocks { get; set; } = 0; + public int peerWants { get; set; } = 0; + public int exchanged { get; set; } = 0; + public string lastExchange { get; set; } = string.Empty; + } + + public class SwitchPeerResponse + { + public string peerId { get; set; } = string.Empty; + public string key { get; set; } = string.Empty; + } + + public class CodexDebugVersionResponse + { + public string version { get; set; } = string.Empty; + public string revision { get; set; } = string.Empty; + } + + public class CodexDebugPeerResponse + { + public bool IsPeerFound { get; set; } + + public string peerId { get; set; } = string.Empty; + public long seqNo { get; set; } + public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty(); + } + + public class CodexDebugPeerAddressResponse + { + public string address { get; set; } = string.Empty; + } + + public class CodexSalesAvailabilityRequest + { + public string size { get; set; } = string.Empty; + public string duration { get; set; } = string.Empty; + public string minPrice { get; set; } = string.Empty; + public string maxCollateral { get; set; } = string.Empty; + } + + public class CodexSalesAvailabilityResponse + { + public string id { get; set; } = string.Empty; + public string size { get; set; } = string.Empty; + public string duration { get; set; } = string.Empty; + public string minPrice { get; set; } = string.Empty; + public string maxCollateral { get; set; } = string.Empty; + } + + public class CodexSalesRequestStorageRequest + { + public string duration { get; set; } = string.Empty; + public string proofProbability { get; set; } = string.Empty; + public string reward { get; set; } = string.Empty; + public string collateral { get; set; } = string.Empty; + public string? expiry { get; set; } + public uint? nodes { get; set; } + public uint? tolerance { get; set; } + } +} diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 8f39381..af76127 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using KubernetesWorkflow; +using Utils; namespace DistTestCore { @@ -52,7 +53,7 @@ namespace DistTestCore return runnerLocation; } - public RunningContainerAddress GetAddress(RunningContainer container) + public Address GetAddress(RunningContainer container) { if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster) { diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index 5ebae09..8d8c55f 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -19,11 +19,11 @@ namespace DistTestCore public const int ChunkSize = 1024 * 1024 * 100; private static NumberSource folderNumberSource = new NumberSource(0); private readonly Random random = new Random(); - private readonly TestLog log; + private readonly BaseLog log; private readonly string folder; private readonly List> fileSetStack = new List>(); - public FileManager(TestLog log, Configuration configuration) + public FileManager(BaseLog log, Configuration configuration) { folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5")); @@ -142,9 +142,9 @@ namespace DistTestCore public class TestFile { - private readonly TestLog log; + private readonly BaseLog log; - public TestFile(TestLog log, string filename, string label) + public TestFile(BaseLog log, string filename, string label) { this.log = log; Filename = filename; diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 30bf48c..2aa5085 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,5 +1,4 @@ -using KubernetesWorkflow; -using Logging; +using Logging; using Newtonsoft.Json; using System.Net.Http.Headers; using System.Net.Http.Json; @@ -11,11 +10,11 @@ namespace DistTestCore { private readonly BaseLog log; private readonly ITimeSet timeSet; - private readonly RunningContainerAddress address; + private readonly Address address; private readonly string baseUrl; private readonly TimeSpan? timeoutOverride; - public Http(BaseLog log, ITimeSet timeSet, RunningContainerAddress address, string baseUrl, TimeSpan? timeoutOverride = null) + public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, TimeSpan? timeoutOverride = null) { this.log = log; this.timeSet = timeSet; diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 7bac8b2..29730a6 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace $"proofProbability: {proofProbability}, " + $"duration: {Time.FormatDuration(duration)})"); - var response = codexAccess.RequestStorage(request, contentId.Id); + var response = codexAccess.Node.RequestStorage(request, contentId.Id); if (response == "Purchasing not available") { @@ -78,7 +78,7 @@ namespace DistTestCore.Marketplace $"maxCollateral: {maxCollateral}, " + $"maxDuration: {Time.FormatDuration(maxDuration)})"); - var response = codexAccess.SalesAvailability(request); + var response = codexAccess.Node.SalesAvailability(request); Log($"Storage successfully made available. Id: {response.id}"); diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index d53f2c2..adc7dfc 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -49,7 +49,7 @@ namespace DistTestCore public CodexDebugResponse GetDebugInfo() { - var debugInfo = CodexAccess.GetDebugInfo(); + var debugInfo = CodexAccess.Node.GetDebugInfo(); var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId)); Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}"); return debugInfo; @@ -57,12 +57,12 @@ namespace DistTestCore public CodexDebugPeerResponse GetDebugPeer(string peerId) { - return CodexAccess.GetDebugPeer(peerId); + return CodexAccess.Node.GetDebugPeer(peerId); } public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) { - return CodexAccess.GetDebugPeer(peerId, timeout); + return CodexAccess.Node.GetDebugPeer(peerId, timeout); } public ContentId UploadFile(TestFile file) @@ -72,7 +72,7 @@ namespace DistTestCore var logMessage = $"Uploading file {file.Describe()}..."; var response = Stopwatch.Measure(lifecycle.Log, logMessage, () => { - return CodexAccess.UploadFile(fileStream); + return CodexAccess.Node.UploadFile(fileStream); }); if (response.StartsWith(UploadFailedMessage)) @@ -101,7 +101,7 @@ namespace DistTestCore Log($"Connecting to peer {peer.GetName()}..."); var peerInfo = node.GetDebugInfo(); - var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo)); + var response = CodexAccess.Node.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo)); Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); Log($"Successfully connected to peer {peer.GetName()}."); @@ -141,7 +141,7 @@ namespace DistTestCore using var fileStream = File.OpenWrite(file.Filename); try { - using var downloadStream = CodexAccess.DownloadFile(contentId); + using var downloadStream = CodexAccess.Node.DownloadFile(contentId); downloadStream.CopyTo(fileStream); } catch diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 0b7b3fd..6e2224d 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -1,4 +1,6 @@ -namespace KubernetesWorkflow +using Utils; + +namespace KubernetesWorkflow { public class RunningContainers { @@ -21,7 +23,7 @@ public class RunningContainer { - public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, RunningContainerAddress clusterExternalAddress, RunningContainerAddress clusterInternalAddress) + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, Address clusterExternalAddress, Address clusterInternalAddress) { Pod = pod; Recipe = recipe; @@ -35,8 +37,8 @@ public RunningPod Pod { get; } public ContainerRecipe Recipe { get; } public Port[] ServicePorts { get; } - public RunningContainerAddress ClusterExternalAddress { get; } - public RunningContainerAddress ClusterInternalAddress { get; } + public Address ClusterExternalAddress { get; } + public Address ClusterInternalAddress { get; } private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig) { @@ -50,16 +52,4 @@ } } } - - public class RunningContainerAddress - { - public RunningContainerAddress(string host, int port) - { - Host = host; - Port = port; - } - - public string Host { get; } - public int Port { get; } - } } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 59fe617..8a223cf 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -1,4 +1,5 @@ using Logging; +using Utils; namespace KubernetesWorkflow { @@ -87,20 +88,20 @@ namespace KubernetesWorkflow }).ToArray(); } - private RunningContainerAddress GetContainerExternalAddress(RunningPod pod, Port[] servicePorts) + private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts) { - return new RunningContainerAddress( + return new Address( pod.Cluster.HostAddress, GetServicePort(servicePorts)); } - private RunningContainerAddress GetContainerInternalAddress(ContainerRecipe recipe) + private Address GetContainerInternalAddress(ContainerRecipe recipe) { var serviceName = "service-" + numberSource.WorkflowNumber; var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace; var port = GetInternalPort(recipe); - return new RunningContainerAddress( + return new Address( $"http://{serviceName}.{namespaceName}.svc.cluster.local", port); } diff --git a/Utils/Address.cs b/Utils/Address.cs new file mode 100644 index 0000000..510afeb --- /dev/null +++ b/Utils/Address.cs @@ -0,0 +1,14 @@ +namespace Utils +{ + public class Address + { + public Address(string host, int port) + { + Host = host; + Port = port; + } + + public string Host { get; } + public int Port { get; } + } +} diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 4bc64fc..7946ff4 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -17,6 +17,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -51,6 +53,10 @@ Global {D6C3555E-D52D-4993-A87B-71AB650398FD}.Debug|Any CPU.Build.0 = Debug|Any CPU {D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.ActiveCfg = Release|Any CPU {D6C3555E-D52D-4993-A87B-71AB650398FD}.Release|Any CPU.Build.0 = Release|Any CPU + {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 978d2d3a84c3a203207d7c30cdf6b2fe145f9754 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 21 Jun 2023 09:27:59 +0200 Subject: [PATCH 02/23] Instantiate node-handles for each individual test. Allows for time-set selection. --- ContinuousTests/CodexNodeFactory.cs | 22 +++++++ ContinuousTests/Configuration.cs | 2 +- ContinuousTests/ContinuousTest.cs | 82 +++++++++++++++++++++++++ ContinuousTests/ContinuousTestRunner.cs | 28 ++++----- ContinuousTests/IContinuousTest.cs | 33 ---------- ContinuousTests/TestFactory.cs | 12 ++++ ContinuousTests/TestFinder.cs | 24 -------- ContinuousTests/TestRun.cs | 44 ++++++------- ContinuousTests/Tests/TwoClientTest.cs | 22 +++++++ 9 files changed, 175 insertions(+), 94 deletions(-) create mode 100644 ContinuousTests/CodexNodeFactory.cs create mode 100644 ContinuousTests/ContinuousTest.cs delete mode 100644 ContinuousTests/IContinuousTest.cs create mode 100644 ContinuousTests/TestFactory.cs delete mode 100644 ContinuousTests/TestFinder.cs create mode 100644 ContinuousTests/Tests/TwoClientTest.cs diff --git a/ContinuousTests/CodexNodeFactory.cs b/ContinuousTests/CodexNodeFactory.cs new file mode 100644 index 0000000..a3bf54b --- /dev/null +++ b/ContinuousTests/CodexNodeFactory.cs @@ -0,0 +1,22 @@ +using DistTestCore; +using DistTestCore.Codex; +using Logging; +using Utils; + +namespace ContinuousTests +{ + public class CodexNodeFactory + { + public CodexNode[] Create(string[] urls, BaseLog log, ITimeSet timeSet) + { + return urls.Select(url => + { + var cutIndex = url.LastIndexOf(':'); + var host = url.Substring(0, cutIndex); + var port = url.Substring(cutIndex + 1); + var address = new Address(host, Convert.ToInt32(port)); + return new CodexNode(log, timeSet, address); + }).ToArray(); + } + } +} diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index 5ccb845..34421ef 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -40,7 +40,7 @@ namespace ContinuousTests if (!string.IsNullOrEmpty(logPath) && !string.IsNullOrEmpty(codexUrls) && !string.IsNullOrEmpty(sleep)) { var urls = codexUrls.Split(';', StringSplitOptions.RemoveEmptyEntries); - var ms = 0; + int ms; if (int.TryParse(sleep, out ms)) { if (urls.Length > 0) diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs new file mode 100644 index 0000000..725f456 --- /dev/null +++ b/ContinuousTests/ContinuousTest.cs @@ -0,0 +1,82 @@ +using DistTestCore; +using DistTestCore.Codex; +using Logging; + +namespace ContinuousTests +{ + public abstract class ContinuousTestLongTimeouts : ContinuousTest + { + public override ITimeSet TimeSet => new LongTimeSet(); + } + + public abstract class ContinuousTest + { + private const string UploadFailedMessage = "Unable to store block"; + + public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager) + { + Nodes = nodes; + Log = log; + FileManager = fileManager; + } + + public CodexNode[] Nodes { get; private set; } = null!; + public BaseLog Log { get; private set; } = null!; + public IFileManager FileManager { get; private set; } = null!; + public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } } + + public abstract int RequiredNumberOfNodes { get; } + + public string Name + { + get + { + return GetType().Name; + } + } + + public abstract void Run(); + + public ContentId? UploadFile(CodexNode node, TestFile file) + { + using var fileStream = File.OpenRead(file.Filename); + + var logMessage = $"Uploading file {file.Describe()}..."; + var response = Stopwatch.Measure(Log, logMessage, () => + { + return node.UploadFile(fileStream); + }); + + if (response.StartsWith(UploadFailedMessage)) + { + return null; + } + Log.Log($"Uploaded file. Received contentId: '{response}'."); + return new ContentId(response); + } + + public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "") + { + var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; + var file = FileManager.CreateEmptyTestFile(fileLabel); + Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file)); + Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'."); + return file; + } + + private void DownloadToFile(CodexNode node, string contentId, TestFile file) + { + using var fileStream = File.OpenWrite(file.Filename); + try + { + using var downloadStream = node.DownloadFile(contentId); + downloadStream.CopyTo(fileStream); + } + catch + { + Log.Log($"Failed to download file '{contentId}'."); + throw; + } + } + } +} diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index caa9fbf..639390c 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -1,14 +1,14 @@ using DistTestCore; using DistTestCore.Codex; using Logging; -using Utils; namespace ContinuousTests { public class ContinuousTestRunner { private readonly ConfigLoader configLoader = new ConfigLoader(); - private readonly TestFinder testFinder = new TestFinder(); + private readonly TestFactory testFactory = new TestFactory(); + private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); public void Run() { @@ -19,13 +19,13 @@ namespace ContinuousTests log.Log("Checking configuration..."); PreflightCheck(config); log.Log("Contacting Codex nodes..."); - var nodes = CreateCodexNodes(log, new LongTimeSet(), config); - log.Log("OK"); + CheckCodexNodes(log, config); + log.Log("All OK."); log.Log(""); while (true) { - var run = new TestRun(config, log, testFinder, nodes); + var run = new TestRun(config, log, testFactory); try { @@ -42,7 +42,7 @@ namespace ContinuousTests private void PreflightCheck(Configuration config) { - var tests = testFinder.GetTests(); + var tests = testFactory.CreateTests(); if (!tests.Any()) { throw new Exception("Unable to find any tests."); @@ -57,20 +57,20 @@ namespace ContinuousTests } } + if (!Directory.Exists(config.LogPath)) + { + Directory.CreateDirectory(config.LogPath); + } + if (errors.Any()) { throw new Exception("Prerun check failed: " + string.Join(", ", errors)); } } - private CodexNode[] CreateCodexNodes(BaseLog log, ITimeSet timeSet, Configuration config) + private void CheckCodexNodes(BaseLog log, Configuration config) { - var nodes = config.CodexUrls.Select(url => - { - var address = new Address(url, 1234); - return new CodexNode(log, timeSet, address); - }).ToArray(); - + var nodes = codexNodeFactory.Create(config.CodexUrls, log, new DefaultTimeSet()); var pass = true; foreach (var n in nodes) { @@ -90,8 +90,6 @@ namespace ContinuousTests { throw new Exception("Not all codex nodes responded."); } - - return nodes; } private bool EnsureOnline(CodexNode n) diff --git a/ContinuousTests/IContinuousTest.cs b/ContinuousTests/IContinuousTest.cs deleted file mode 100644 index 0374ffd..0000000 --- a/ContinuousTests/IContinuousTest.cs +++ /dev/null @@ -1,33 +0,0 @@ -using DistTestCore; -using DistTestCore.Codex; -using Logging; - -namespace ContinuousTests -{ - public interface IContinuousTest - { - string Name { get; } - int RequiredNumberOfNodes { get; } - - void Run(); - } - - public abstract class ContinuousTest : IContinuousTest - { - public CodexNode[] Nodes { get; set; } = null!; - public BaseLog Log { get; set; } = null!; - public FileManager FileManager { get; set; } = null!; - - public abstract int RequiredNumberOfNodes { get; } - - public string Name - { - get - { - return GetType().Name; - } - } - - public abstract void Run(); - } -} diff --git a/ContinuousTests/TestFactory.cs b/ContinuousTests/TestFactory.cs new file mode 100644 index 0000000..3a4992a --- /dev/null +++ b/ContinuousTests/TestFactory.cs @@ -0,0 +1,12 @@ +namespace ContinuousTests +{ + public class TestFactory + { + public ContinuousTest[] CreateTests() + { + var types = GetType().Assembly.GetTypes(); + var testTypes = types.Where(t => typeof(ContinuousTest).IsAssignableFrom(t) && !t.IsAbstract); + return testTypes.Select(t => (ContinuousTest)Activator.CreateInstance(t)!).ToArray(); + } + } +} diff --git a/ContinuousTests/TestFinder.cs b/ContinuousTests/TestFinder.cs deleted file mode 100644 index 349e957..0000000 --- a/ContinuousTests/TestFinder.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace ContinuousTests -{ - public class TestFinder - { - private readonly List testList = new List(); - - public IContinuousTest[] GetTests() - { - if (!testList.Any()) FindTests(); - return testList.ToArray(); - } - - private void FindTests() - { - var types = GetType().Assembly.GetTypes(); - var testTypes = types.Where(t => typeof(IContinuousTest).IsAssignableFrom(t) && !t.IsAbstract); - foreach (var testType in testTypes) - { - var t = Activator.CreateInstance(testType); - testList.Add((IContinuousTest)t!); - } - } - } -} diff --git a/ContinuousTests/TestRun.cs b/ContinuousTests/TestRun.cs index 9138744..43e71b0 100644 --- a/ContinuousTests/TestRun.cs +++ b/ContinuousTests/TestRun.cs @@ -7,29 +7,30 @@ namespace ContinuousTests public class TestRun { private readonly Random random = new Random(); + private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); private readonly Configuration config; private readonly BaseLog log; - private readonly TestFinder testFinder; - private readonly CodexNode[] nodes; + private readonly TestFactory testFinder; private readonly FileManager fileManager; + private ITimeSet timeSet; - public TestRun(Configuration config, BaseLog log, TestFinder testFinder, CodexNode[] nodes) + public TestRun(Configuration config, BaseLog log, TestFactory testFinder) { this.config = config; this.log = log; this.testFinder = testFinder; - this.nodes = nodes; fileManager = new FileManager(log, new DistTestCore.Configuration()); + timeSet = new DefaultTimeSet(); } public void Run() { - var remainingTests = testFinder.GetTests().ToList(); + var remainingTests = testFinder.CreateTests().ToList(); while (remainingTests.Any()) { var test = PickOneRandom(remainingTests); - var selectedNodes = SelectRandomNodes(test.RequiredNumberOfNodes); - AssignEssentials(test, selectedNodes); + var nodes = CreateRandomNodes(test.RequiredNumberOfNodes); + AssignEssentials(test, nodes); fileManager.PushFileSet(); log.Log($"Start '{test.Name}'"); @@ -49,33 +50,34 @@ namespace ContinuousTests } } - private void AssignEssentials(IContinuousTest test, CodexNode[] nodes) + private void AssignEssentials(ContinuousTest test, CodexNode[] nodes) { - var t = (ContinuousTest)test; - t.Nodes = nodes; - t.Log = log; - t.FileManager = fileManager; + test.Initialize(nodes, log, fileManager); } - private void ClearEssentials(IContinuousTest test) + private void ClearEssentials(ContinuousTest test) { - var t = (ContinuousTest)test; - t.Nodes = null!; - t.Log = null!; - t.FileManager = null!; + // Looks a little strange, but prevents finished test from interacting further. + test.Initialize(null!, null!, null!); } - private CodexNode[] SelectRandomNodes(int number) + private string[] SelectRandomUrls(int number) { - var remainingNodes = nodes.ToList(); - var result = new CodexNode[number]; + var urls = config.CodexUrls.ToList(); + var result = new string[number]; for (var i = 0; i < number; i++) { - result[i] = PickOneRandom(remainingNodes); + result[i] = PickOneRandom(urls); } return result; } + private CodexNode[] CreateRandomNodes(int number) + { + var urls = SelectRandomUrls(number); + return codexNodeFactory.Create(urls, log, timeSet); + } + private T PickOneRandom(List remainingItems) { var i = random.Next(0, remainingItems.Count); diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs new file mode 100644 index 0000000..c38bc92 --- /dev/null +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -0,0 +1,22 @@ +using DistTestCore; +using NUnit.Framework; + +namespace ContinuousTests.Tests +{ + public class TwoClientTest : ContinuousTest + { + public override int RequiredNumberOfNodes => 2; + + public override void Run() + { + var file = FileManager.GenerateTestFile(10.MB()); + + var cid = UploadFile(Nodes[0], file); + Assert.That(cid, Is.Not.Null); + + var dl = DownloadContent(Nodes[1], cid!); + + dl.AssertIsEqual(file); + } + } +} From c5c54f5963e0fd258eb58a95b7dfd48af0606942 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 21 Jun 2023 10:06:54 +0200 Subject: [PATCH 03/23] Better logging alignment for continous tests --- .../{TestRun.cs => AllTestsRun.cs} | 80 ++++++++++--------- ContinuousTests/Configuration.cs | 4 + ContinuousTests/ContinuousTestRunner.cs | 38 ++++++--- Logging/FixtureLog.cs | 12 +-- Logging/TestLog.cs | 8 +- Utils/RandomUtils.cs | 15 ++++ 6 files changed, 102 insertions(+), 55 deletions(-) rename ContinuousTests/{TestRun.cs => AllTestsRun.cs} (50%) create mode 100644 Utils/RandomUtils.cs diff --git a/ContinuousTests/TestRun.cs b/ContinuousTests/AllTestsRun.cs similarity index 50% rename from ContinuousTests/TestRun.cs rename to ContinuousTests/AllTestsRun.cs index 43e71b0..3fc0d4a 100644 --- a/ContinuousTests/TestRun.cs +++ b/ContinuousTests/AllTestsRun.cs @@ -1,64 +1,86 @@ using DistTestCore; using DistTestCore.Codex; using Logging; +using Utils; namespace ContinuousTests { - public class TestRun + public class AllTestsRun { - private readonly Random random = new Random(); - private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); private readonly Configuration config; - private readonly BaseLog log; + private readonly FixtureLog log; private readonly TestFactory testFinder; - private readonly FileManager fileManager; - private ITimeSet timeSet; - public TestRun(Configuration config, BaseLog log, TestFactory testFinder) + public AllTestsRun(Configuration config, FixtureLog log, TestFactory testFinder) { this.config = config; this.log = log; this.testFinder = testFinder; - fileManager = new FileManager(log, new DistTestCore.Configuration()); - timeSet = new DefaultTimeSet(); } - public void Run() + public ContinuousTestResult RunAll() { var remainingTests = testFinder.CreateTests().ToList(); + var result = ContinuousTestResult.Passed; while (remainingTests.Any()) { - var test = PickOneRandom(remainingTests); - var nodes = CreateRandomNodes(test.RequiredNumberOfNodes); - AssignEssentials(test, nodes); - fileManager.PushFileSet(); + var test = remainingTests.PickOneRandom(); + var testLog = log.CreateTestLog(test.Name); + var singleTestRun = new SingleTestRun(config, test, testLog); log.Log($"Start '{test.Name}'"); try { - test.Run(); + singleTestRun.Run(); log.Log($"'{test.Name}' = Passed"); } catch { log.Log($"'{test.Name}' = Failed"); + testLog.MarkAsFailed(); + result = ContinuousTestResult.Failed; } - fileManager.PopFileSet(); - ClearEssentials(test); Thread.Sleep(config.SleepSecondsPerTest * 1000); } + return result; + } + } + + public class SingleTestRun + { + private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); + private readonly Configuration config; + private readonly ContinuousTest test; + private readonly CodexNode[] nodes; + private readonly FileManager fileManager; + + public SingleTestRun(Configuration config, ContinuousTest test, BaseLog testLog) + { + this.config = config; + this.test = test; + + nodes = CreateRandomNodes(test.RequiredNumberOfNodes, testLog); + fileManager = new FileManager(testLog, new DistTestCore.Configuration()); + + test.Initialize(nodes, testLog, fileManager); } - private void AssignEssentials(ContinuousTest test, CodexNode[] nodes) + public void Run() { - test.Initialize(nodes, log, fileManager); + test.Run(); } - private void ClearEssentials(ContinuousTest test) + public void TearDown() { - // Looks a little strange, but prevents finished test from interacting further. test.Initialize(null!, null!, null!); + fileManager.DeleteAllTestFiles(); + } + + private CodexNode[] CreateRandomNodes(int number, BaseLog testLog) + { + var urls = SelectRandomUrls(number); + return codexNodeFactory.Create(urls, testLog, test.TimeSet); } private string[] SelectRandomUrls(int number) @@ -67,23 +89,9 @@ namespace ContinuousTests var result = new string[number]; for (var i = 0; i < number; i++) { - result[i] = PickOneRandom(urls); + result[i] = urls.PickOneRandom(); } return result; } - - private CodexNode[] CreateRandomNodes(int number) - { - var urls = SelectRandomUrls(number); - return codexNodeFactory.Create(urls, log, timeSet); - } - - private T PickOneRandom(List remainingItems) - { - var i = random.Next(0, remainingItems.Count); - var result = remainingItems[i]; - remainingItems.RemoveAt(i); - return result; - } } } diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index 34421ef..49aca0f 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -16,6 +16,10 @@ namespace ContinuousTests public Configuration Load() { var config = Read(); + //config.LogPath = "logs"; + //config.SleepSecondsPerTest = 10; + //config.CodexUrls = new string[] { "http://localhost:8080", "http://localhost:8081" }; + Validate(config); return config; } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 639390c..b1b9d44 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -13,33 +13,43 @@ namespace ContinuousTests public void Run() { var config = configLoader.Load(); - var log = new TestLog(config.LogPath, true); - - log.Log("Starting continuous test run..."); - log.Log("Checking configuration..."); - PreflightCheck(config); - log.Log("Contacting Codex nodes..."); - CheckCodexNodes(log, config); - log.Log("All OK."); - log.Log(""); + StartupChecks(config); while (true) { - var run = new TestRun(config, log, testFactory); + var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks"); + var allTestsRun = new AllTestsRun(config, log, testFactory); + var result = ContinuousTestResult.Passed; try { - run.Run(); + result = allTestsRun.RunAll(); } catch (Exception ex) { log.Error($"Exception during test run: " + ex); } + if (result == ContinuousTestResult.Failed) + { + log.MarkAsFailed(); + } + Thread.Sleep(config.SleepSecondsPerTest * 1000); } } + private void StartupChecks(Configuration config) + { + var log = new FixtureLog(new LogConfig(config.LogPath, false), "ContinuousTestsRun"); + log.Log("Starting continuous test run..."); + log.Log("Checking configuration..."); + PreflightCheck(config); + log.Log("Contacting Codex nodes..."); + CheckCodexNodes(log, config); + log.Log("All OK."); + } + private void PreflightCheck(Configuration config) { var tests = testFactory.CreateTests(); @@ -106,4 +116,10 @@ namespace ContinuousTests return true; } } + + public enum ContinuousTestResult + { + Passed, + Failed + } } diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs index 1502ec9..53eae10 100644 --- a/Logging/FixtureLog.cs +++ b/Logging/FixtureLog.cs @@ -8,19 +8,19 @@ namespace Logging private readonly string fullName; private readonly LogConfig config; - public FixtureLog(LogConfig config) + public FixtureLog(LogConfig config, string name = "") : base(config.DebugEnabled) { start = DateTime.UtcNow; var folder = DetermineFolder(config); - var fixtureName = GetFixtureName(); + var fixtureName = GetFixtureName(name); fullName = Path.Combine(folder, fixtureName); this.config = config; } - public TestLog CreateTestLog() + public TestLog CreateTestLog(string name = "") { - return new TestLog(fullName, config.DebugEnabled); + return new TestLog(fullName, config.DebugEnabled, name); } protected override LogFile CreateLogFile() @@ -36,10 +36,12 @@ namespace Logging Pad(start.Day)); } - private string GetFixtureName() + private string GetFixtureName(string name) { var test = TestContext.CurrentContext.Test; var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); + if (!string.IsNullOrEmpty(name)) className = name; + return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}"; } diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 39f70a0..6ac0c99 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -9,10 +9,10 @@ namespace Logging private readonly string methodName; private readonly string fullName; - public TestLog(string folder, bool debug) + public TestLog(string folder, bool debug, string name = "") : base(debug) { - methodName = GetMethodName(); + methodName = GetMethodName(name); fullName = Path.Combine(folder, methodName); Log($"*** Begin: {methodName}"); @@ -39,13 +39,15 @@ namespace Logging MarkAsFailed(); } } + protected override LogFile CreateLogFile() { return new LogFile(fullName, "log"); } - private string GetMethodName() + private string GetMethodName(string name) { + if (!string.IsNullOrEmpty(name)) return name; var test = TestContext.CurrentContext.Test; var args = FormatArguments(test); return $"{test.MethodName}{args}"; diff --git a/Utils/RandomUtils.cs b/Utils/RandomUtils.cs new file mode 100644 index 0000000..0cf0132 --- /dev/null +++ b/Utils/RandomUtils.cs @@ -0,0 +1,15 @@ +namespace Utils +{ + public static class RandomUtils + { + private static readonly Random random = new Random(); + + public static T PickOneRandom(this List remainingItems) + { + var i = random.Next(0, remainingItems.Count); + var result = remainingItems[i]; + remainingItems.RemoveAt(i); + return result; + } + } +} From 0d9918b401fd71757236c5ff92c2a570817c97fd Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 21 Jun 2023 10:34:29 +0200 Subject: [PATCH 04/23] Sleep per test and per all-tests --- ContinuousTests/AllTestsRun.cs | 2 +- ContinuousTests/Configuration.cs | 48 +++++++++++++++++-------- ContinuousTests/ContinuousTestRunner.cs | 6 ++-- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/ContinuousTests/AllTestsRun.cs b/ContinuousTests/AllTestsRun.cs index 3fc0d4a..661b6af 100644 --- a/ContinuousTests/AllTestsRun.cs +++ b/ContinuousTests/AllTestsRun.cs @@ -41,7 +41,7 @@ namespace ContinuousTests result = ContinuousTestResult.Failed; } - Thread.Sleep(config.SleepSecondsPerTest * 1000); + Thread.Sleep(config.SleepSecondsPerSingleTest * 1000); } return result; } diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index 49aca0f..e51f4b9 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -6,7 +6,8 @@ namespace ContinuousTests { public string LogPath { get; set; } = string.Empty; public string[] CodexUrls { get; set; } = Array.Empty(); - public int SleepSecondsPerTest { get; set; } + public int SleepSecondsPerSingleTest { get; set; } + public int SleepSecondsPerAllTests { get; set; } } public class ConfigLoader @@ -16,9 +17,6 @@ namespace ContinuousTests public Configuration Load() { var config = Read(); - //config.LogPath = "logs"; - //config.SleepSecondsPerTest = 10; - //config.CodexUrls = new string[] { "http://localhost:8080", "http://localhost:8081" }; Validate(config); return config; @@ -39,33 +37,53 @@ namespace ContinuousTests var logPath = Environment.GetEnvironmentVariable("LOGPATH"); var codexUrls = Environment.GetEnvironmentVariable("CODEXURLS"); - var sleep = Environment.GetEnvironmentVariable("SLEEPSECONDSPERTEST"); + var sleepPerSingle = Environment.GetEnvironmentVariable("SLEEPSECONDSPERSINGLETEST"); + var sleepPerAll = Environment.GetEnvironmentVariable("SLEEPSECONDSPERALLTESTS"); - if (!string.IsNullOrEmpty(logPath) && !string.IsNullOrEmpty(codexUrls) && !string.IsNullOrEmpty(sleep)) + if (!string.IsNullOrEmpty(logPath) && + !string.IsNullOrEmpty(codexUrls) && + !string.IsNullOrEmpty(sleepPerSingle) && + !string.IsNullOrEmpty(sleepPerAll)) { var urls = codexUrls.Split(';', StringSplitOptions.RemoveEmptyEntries); - int ms; - if (int.TryParse(sleep, out ms)) + int secondsSingle; + int secondsAll; + if (int.TryParse(sleepPerSingle, out secondsSingle) && int.TryParse(sleepPerAll, out secondsAll)) { if (urls.Length > 0) { - return new Configuration { LogPath = logPath, CodexUrls = urls, SleepSecondsPerTest = ms }; + return new Configuration + { + LogPath = logPath, + CodexUrls = urls, + SleepSecondsPerSingleTest = secondsSingle, + SleepSecondsPerAllTests = secondsAll + }; } } } + var nl = Environment.NewLine; throw new Exception($"Unable to load configuration from '{filename}', and " + - $"unable to load configuration from environment variables 'LOGPATH' and 'CODEXURLS', and 'SLEEPSECONDSPERTEST'. " + - $"(semi-colon-separated URLs) " + - $"Create the configuration file or set the environment veriables."); + $"unable to load configuration from environment variables. " + nl + + $"'LOGPATH' = Path where log files will be saved." + nl + + $"'CODEXURLS' = Semi-colon separated URLs to codex APIs. e.g. 'https://hostaddr_one:port;https://hostaddr_two:port'" + nl + + $"'SLEEPSECONDSPERSINGLETEST' = Seconds to sleep after each individual test." + nl + + $"'SLEEPSECONDSPERALLTESTS' = Seconds to sleep after all tests, before starting again." + nl + + nl); } private void Validate(Configuration configuration) { - if (configuration.SleepSecondsPerTest < 10) + if (configuration.SleepSecondsPerSingleTest < 1) { - Console.WriteLine("Warning: configuration.SleepMsPerTest was less than 10 seconds. Using 10 seconds instead!"); - configuration.SleepSecondsPerTest = 10; + Console.WriteLine("Warning: configuration.SleepSecondsPerSingleTest was less than 1 seconds. Using 1 seconds instead!"); + configuration.SleepSecondsPerSingleTest = 1; + } + if (configuration.SleepSecondsPerAllTests < 1) + { + Console.WriteLine("Warning: configuration.SleepSecondsPerAllTests was less than 10 seconds. Using 10 seconds instead!"); + configuration.SleepSecondsPerAllTests = 10; } if (string.IsNullOrEmpty(configuration.LogPath)) diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index b1b9d44..bc03aca 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -17,7 +17,7 @@ namespace ContinuousTests while (true) { - var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks"); + var log = new FixtureLog(new LogConfig(config.LogPath, false), "ContinuousTestsRun"); var allTestsRun = new AllTestsRun(config, log, testFactory); var result = ContinuousTestResult.Passed; @@ -35,13 +35,13 @@ namespace ContinuousTests log.MarkAsFailed(); } - Thread.Sleep(config.SleepSecondsPerTest * 1000); + Thread.Sleep(config.SleepSecondsPerSingleTest * 1000); } } private void StartupChecks(Configuration config) { - var log = new FixtureLog(new LogConfig(config.LogPath, false), "ContinuousTestsRun"); + var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks"); log.Log("Starting continuous test run..."); log.Log("Checking configuration..."); PreflightCheck(config); From ae5309c2b69b154d48bfad35e35489e97c925b01 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 21 Jun 2023 11:01:48 +0200 Subject: [PATCH 05/23] Deletes log files for tests and runs that passed. --- ContinuousTests/AllTestsRun.cs | 52 +---------------------- ContinuousTests/Configuration.cs | 16 ++++--- ContinuousTests/ContinuousTestRunner.cs | 4 ++ ContinuousTests/SingleTestRun.cs | 55 +++++++++++++++++++++++++ Logging/BaseLog.cs | 5 +++ Logging/FixtureLog.cs | 5 +++ 6 files changed, 81 insertions(+), 56 deletions(-) create mode 100644 ContinuousTests/SingleTestRun.cs diff --git a/ContinuousTests/AllTestsRun.cs b/ContinuousTests/AllTestsRun.cs index 661b6af..d825cbe 100644 --- a/ContinuousTests/AllTestsRun.cs +++ b/ContinuousTests/AllTestsRun.cs @@ -1,6 +1,4 @@ -using DistTestCore; -using DistTestCore.Codex; -using Logging; +using Logging; using Utils; namespace ContinuousTests @@ -33,6 +31,7 @@ namespace ContinuousTests { singleTestRun.Run(); log.Log($"'{test.Name}' = Passed"); + if (!config.KeepPassedTestLogs) testLog.Delete(); } catch { @@ -43,54 +42,7 @@ namespace ContinuousTests Thread.Sleep(config.SleepSecondsPerSingleTest * 1000); } - return result; - } - } - public class SingleTestRun - { - private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); - private readonly Configuration config; - private readonly ContinuousTest test; - private readonly CodexNode[] nodes; - private readonly FileManager fileManager; - - public SingleTestRun(Configuration config, ContinuousTest test, BaseLog testLog) - { - this.config = config; - this.test = test; - - nodes = CreateRandomNodes(test.RequiredNumberOfNodes, testLog); - fileManager = new FileManager(testLog, new DistTestCore.Configuration()); - - test.Initialize(nodes, testLog, fileManager); - } - - public void Run() - { - test.Run(); - } - - public void TearDown() - { - test.Initialize(null!, null!, null!); - fileManager.DeleteAllTestFiles(); - } - - private CodexNode[] CreateRandomNodes(int number, BaseLog testLog) - { - var urls = SelectRandomUrls(number); - return codexNodeFactory.Create(urls, testLog, test.TimeSet); - } - - private string[] SelectRandomUrls(int number) - { - var urls = config.CodexUrls.ToList(); - var result = new string[number]; - for (var i = 0; i < number; i++) - { - result[i] = urls.PickOneRandom(); - } return result; } } diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index e51f4b9..4e6278e 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -8,6 +8,7 @@ namespace ContinuousTests public string[] CodexUrls { get; set; } = Array.Empty(); public int SleepSecondsPerSingleTest { get; set; } public int SleepSecondsPerAllTests { get; set; } + public bool KeepPassedTestLogs { get; set; } } public class ConfigLoader @@ -39,6 +40,7 @@ namespace ContinuousTests var codexUrls = Environment.GetEnvironmentVariable("CODEXURLS"); var sleepPerSingle = Environment.GetEnvironmentVariable("SLEEPSECONDSPERSINGLETEST"); var sleepPerAll = Environment.GetEnvironmentVariable("SLEEPSECONDSPERALLTESTS"); + var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS"); if (!string.IsNullOrEmpty(logPath) && !string.IsNullOrEmpty(codexUrls) && @@ -57,7 +59,8 @@ namespace ContinuousTests LogPath = logPath, CodexUrls = urls, SleepSecondsPerSingleTest = secondsSingle, - SleepSecondsPerAllTests = secondsAll + SleepSecondsPerAllTests = secondsAll, + KeepPassedTestLogs = keep == "1" }; } } @@ -65,11 +68,12 @@ namespace ContinuousTests var nl = Environment.NewLine; throw new Exception($"Unable to load configuration from '{filename}', and " + - $"unable to load configuration from environment variables. " + nl + - $"'LOGPATH' = Path where log files will be saved." + nl + - $"'CODEXURLS' = Semi-colon separated URLs to codex APIs. e.g. 'https://hostaddr_one:port;https://hostaddr_two:port'" + nl + - $"'SLEEPSECONDSPERSINGLETEST' = Seconds to sleep after each individual test." + nl + - $"'SLEEPSECONDSPERALLTESTS' = Seconds to sleep after all tests, before starting again." + nl + + "unable to load configuration from environment variables. " + nl + + "'LOGPATH' = Path where log files will be saved." + nl + + "'CODEXURLS' = Semi-colon separated URLs to codex APIs. e.g. 'https://hostaddr_one:port;https://hostaddr_two:port'" + nl + + "'SLEEPSECONDSPERSINGLETEST' = Seconds to sleep after each individual test." + nl + + "'SLEEPSECONDSPERALLTESTS' = Seconds to sleep after all tests, before starting again." + nl + + "'KEEPPASSEDTESTLOGS' = (Optional, default: 0) Set to '1' to keep log files of tests that passed." + nl + nl); } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index bc03aca..c829a1c 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -34,6 +34,10 @@ namespace ContinuousTests { log.MarkAsFailed(); } + if (!config.KeepPassedTestLogs && result == ContinuousTestResult.Passed) + { + log.DeleteFolder(); + } Thread.Sleep(config.SleepSecondsPerSingleTest * 1000); } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs new file mode 100644 index 0000000..e7c39a7 --- /dev/null +++ b/ContinuousTests/SingleTestRun.cs @@ -0,0 +1,55 @@ +using DistTestCore.Codex; +using DistTestCore; +using Logging; +using Utils; + +namespace ContinuousTests +{ + public class SingleTestRun + { + private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); + private readonly Configuration config; + private readonly ContinuousTest test; + private readonly CodexNode[] nodes; + private readonly FileManager fileManager; + + public SingleTestRun(Configuration config, ContinuousTest test, BaseLog testLog) + { + this.config = config; + this.test = test; + + nodes = CreateRandomNodes(test.RequiredNumberOfNodes, testLog); + fileManager = new FileManager(testLog, new DistTestCore.Configuration()); + + test.Initialize(nodes, testLog, fileManager); + } + + public void Run() + { + test.Run(); + } + + public void TearDown() + { + test.Initialize(null!, null!, null!); + fileManager.DeleteAllTestFiles(); + } + + private CodexNode[] CreateRandomNodes(int number, BaseLog testLog) + { + var urls = SelectRandomUrls(number); + return codexNodeFactory.Create(urls, testLog, test.TimeSet); + } + + private string[] SelectRandomUrls(int number) + { + var urls = config.CodexUrls.ToList(); + var result = new string[number]; + for (var i = 0; i < number; i++) + { + result[i] = urls.PickOneRandom(); + } + return result; + } + } +} diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs index 6a35597..2c69f9c 100644 --- a/Logging/BaseLog.cs +++ b/Logging/BaseLog.cs @@ -58,6 +58,11 @@ namespace Logging replacements.Add(new BaseLogStringReplacement(from, to)); } + public void Delete() + { + File.Delete(LogFile.FullFilename); + } + private string ApplyReplacements(string str) { foreach (var replacement in replacements) diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs index 53eae10..cc5ea87 100644 --- a/Logging/FixtureLog.cs +++ b/Logging/FixtureLog.cs @@ -23,6 +23,11 @@ namespace Logging return new TestLog(fullName, config.DebugEnabled, name); } + public void DeleteFolder() + { + Directory.Delete(fullName, true); + } + protected override LogFile CreateLogFile() { return new LogFile(fullName, "log"); From 8d2d2d6197e42c9a3278748ec69f6fdc881f7243 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 21 Jun 2023 11:45:45 +0200 Subject: [PATCH 06/23] Performance tests --- ContinuousTests/ContinuousTestRunner.cs | 10 ++- ContinuousTests/SingleTestRun.cs | 1 + ContinuousTests/Tests/PerformanceTests.cs | 80 +++++++++++++++++++++++ ContinuousTests/Tests/TwoClientTest.cs | 2 +- 4 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 ContinuousTests/Tests/PerformanceTests.cs diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index c829a1c..cb6bc3f 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -12,7 +12,15 @@ namespace ContinuousTests public void Run() { - var config = configLoader.Load(); + var config = //configLoader.Load(); + new Configuration + { + CodexUrls =new[] { "http://localhost:8080", "http://localhost:8081" }, + LogPath = "logs", + KeepPassedTestLogs = false, + SleepSecondsPerAllTests = 1, + SleepSecondsPerSingleTest = 1, + }; StartupChecks(config); while (true) diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index e7c39a7..978f4bd 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -38,6 +38,7 @@ namespace ContinuousTests private CodexNode[] CreateRandomNodes(int number, BaseLog testLog) { var urls = SelectRandomUrls(number); + testLog.Log("Selected nodes: " + string.Join(",", urls)); return codexNodeFactory.Create(urls, testLog, test.TimeSet); } diff --git a/ContinuousTests/Tests/PerformanceTests.cs b/ContinuousTests/Tests/PerformanceTests.cs new file mode 100644 index 0000000..f0ee853 --- /dev/null +++ b/ContinuousTests/Tests/PerformanceTests.cs @@ -0,0 +1,80 @@ +using DistTestCore; +using DistTestCore.Codex; +using NUnit.Framework; + +namespace ContinuousTests.Tests +{ + public class UploadPerformanceTest : PerformanceTest + { + public override int RequiredNumberOfNodes => 1; + + public override void Run() + { + UploadTest(100, Nodes[0]); + } + } + + public class DownloadLocalPerformanceTest : PerformanceTest + { + public override int RequiredNumberOfNodes => 1; + + public override void Run() + { + DownloadTest(100, Nodes[0], Nodes[0]); + } + } + + public class DownloadRemotePerformanceTest : PerformanceTest + { + public override int RequiredNumberOfNodes => 2; + + public override void Run() + { + DownloadTest(100, Nodes[0], Nodes[1]); + } + } + + public abstract class PerformanceTest : ContinuousTest + { + public void UploadTest(int megabytes, CodexNode uploadNode) + { + var file = FileManager.GenerateTestFile(megabytes.MB()); + + var time = Measure(() => + { + UploadFile(uploadNode, file); + }); + + var timePerMB = time / megabytes; + + Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); + } + + public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode) + { + var file = FileManager.GenerateTestFile(megabytes.MB()); + + var cid = UploadFile(uploadNode, file); + Assert.That(cid, Is.Not.Null); + + TestFile? result = null; + var time = Measure(() => + { + result = DownloadContent(downloadNode, cid!); + }); + + file.AssertIsEqual(result); + + var timePerMB = time / megabytes; + + Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); + } + + private static TimeSpan Measure(Action action) + { + var start = DateTime.UtcNow; + action(); + return DateTime.UtcNow - start; + } + } +} diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index c38bc92..07fa868 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -16,7 +16,7 @@ namespace ContinuousTests.Tests var dl = DownloadContent(Nodes[1], cid!); - dl.AssertIsEqual(file); + file.AssertIsEqual(dl); } } } From 40046d9bcb46b063ea3eae2fda2d5f480adb6b71 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 08:29:33 +0200 Subject: [PATCH 07/23] New console app --- CodexNetDeployer/CodexNetDeployer.csproj | 10 ++++++++++ CodexNetDeployer/Program.cs | 7 +++++++ cs-codex-dist-testing.sln | 8 +++++++- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 CodexNetDeployer/CodexNetDeployer.csproj create mode 100644 CodexNetDeployer/Program.cs diff --git a/CodexNetDeployer/CodexNetDeployer.csproj b/CodexNetDeployer/CodexNetDeployer.csproj new file mode 100644 index 0000000..f02677b --- /dev/null +++ b/CodexNetDeployer/CodexNetDeployer.csproj @@ -0,0 +1,10 @@ + + + + Exe + net7.0 + enable + enable + + + diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs new file mode 100644 index 0000000..a08476f --- /dev/null +++ b/CodexNetDeployer/Program.cs @@ -0,0 +1,7 @@ +public class Program +{ + public static void Main(string[] args) + { + Console.WriteLine("Hello, World!"); + } +} diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 7946ff4..cec7e4c 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -17,7 +17,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Logging\Logging. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NethereumWorkflow", "Nethereum\NethereumWorkflow.csproj", "{D6C3555E-D52D-4993-A87B-71AB650398FD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ContinuousTests", "ContinuousTests\ContinuousTests.csproj", "{025B7074-0A09-4FCC-9BB9-03AE2A961EA1}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDeployer", "CodexNetDeployer\CodexNetDeployer.csproj", "{871CAF12-14BE-4509-BC6E-20FDF0B1083A}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -57,6 +59,10 @@ Global {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Debug|Any CPU.Build.0 = Debug|Any CPU {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.ActiveCfg = Release|Any CPU {025B7074-0A09-4FCC-9BB9-03AE2A961EA1}.Release|Any CPU.Build.0 = Release|Any CPU + {871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {871CAF12-14BE-4509-BC6E-20FDF0B1083A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From e5a7f04c4e175c252a466e2a5180102fe15db170 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 09:51:25 +0200 Subject: [PATCH 08/23] Arg and env-var parsing --- CodexNetDeployer/ArgOrVar.cs | 89 ++++++++++++++++++++++++ CodexNetDeployer/CodexNetDeployer.csproj | 4 ++ CodexNetDeployer/Configuration.cs | 84 ++++++++++++++++++++++ CodexNetDeployer/Program.cs | 43 +++++++++++- 4 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 CodexNetDeployer/ArgOrVar.cs create mode 100644 CodexNetDeployer/Configuration.cs diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs new file mode 100644 index 0000000..df70171 --- /dev/null +++ b/CodexNetDeployer/ArgOrVar.cs @@ -0,0 +1,89 @@ +namespace CodexNetDeployer +{ + public class ArgOrVar + { + public static readonly ArgVar CodexImage = new ArgVar("codex-image", "CODEXIMAGE", "Docker image of Codex."); + public static readonly ArgVar GethImage = new ArgVar("geth-image", "GETHIMAGE", "Docker image of Geth."); + public static readonly ArgVar ContractsImage = new ArgVar("contracts-image", "CONTRACTSIMAGE", "Docker image of Codex Contracts deployer."); + public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file."); + public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment."); + public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created."); + public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node."); + + private readonly string[] args; + + public ArgOrVar(string[] args) + { + this.args = args; + } + + public string Get(ArgVar key, string defaultValue = "") + { + var argKey = $"--{key.Arg}="; + var arg = args.FirstOrDefault(a => a.StartsWith(argKey)); + if (arg != null) + { + return arg.Substring(argKey.Length); + } + + var env = Environment.GetEnvironmentVariable(key.Var); + if (env != null) + { + return env; + } + + return defaultValue; + } + + public int? GetInt(ArgVar key) + { + var str = Get(key); + if (string.IsNullOrEmpty(str)) return null; + if (int.TryParse(str, out int result)) + { + return result; + } + return null; + } + + public void PrintHelp() + { + var nl = Environment.NewLine; + Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " + + "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); + + + Console.Write("\t[ CLI argument ] or [ Environment variable ]"); + Console.CursorLeft = 70; + Console.Write("(Description)" + nl); + var fields = GetType().GetFields();// System.Reflection.BindingFlags.Public & System.Reflection.BindingFlags.Static); + foreach (var field in fields) + { + var value = (ArgVar)field.GetValue(null)!; + value.PrintHelp(); + } + } + } + + public class ArgVar + { + public ArgVar(string arg, string var, string description) + { + Arg = arg; + Var = var; + Description = description; + } + + public string Arg { get; } + public string Var { get; } + public string Description { get; } + + public void PrintHelp() + { + Console.Write($"\t[ --{Arg}=... ] or [ {Var}=... ]"); + Console.CursorLeft = 70; + Console.Write(Description + Environment.NewLine); + } + } +} diff --git a/CodexNetDeployer/CodexNetDeployer.csproj b/CodexNetDeployer/CodexNetDeployer.csproj index f02677b..80f6f4c 100644 --- a/CodexNetDeployer/CodexNetDeployer.csproj +++ b/CodexNetDeployer/CodexNetDeployer.csproj @@ -7,4 +7,8 @@ enable + + + + diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs new file mode 100644 index 0000000..3abb7a4 --- /dev/null +++ b/CodexNetDeployer/Configuration.cs @@ -0,0 +1,84 @@ +namespace CodexNetDeployer +{ + public class Configuration + { + public Configuration( + string codexImage, + string gethImage, + string contractsImage, + string kubeConfigFile, + string kubeNamespace, + int? numberOfCodexNodes, + int? storageQuota) + { + CodexImage = codexImage; + GethImage = gethImage; + ContractsImage = contractsImage; + KubeConfigFile = kubeConfigFile; + KubeNamespace = kubeNamespace; + NumberOfCodexNodes = numberOfCodexNodes; + StorageQuota = storageQuota; + } + + public string CodexImage { get; } + public string GethImage { get; } + public string ContractsImage { get; } + public string KubeConfigFile { get; } + public string KubeNamespace { get; } + public int? NumberOfCodexNodes { get; } + public int? StorageQuota { get; } + + public void PrintConfig() + { + ForEachProperty(onString: Print, onInt: Print); + } + + public List Validate() + { + var errors = new List(); + + ForEachProperty( + onString: (n, v) => StringIsSet(n, v, errors), + onInt: (n, v) => IntIsOverZero(n, v, errors)); + + return errors; + } + + private void ForEachProperty(Action onString, Action onInt) + { + var properties = GetType().GetProperties(); + foreach (var p in properties) + { + if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!); + if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!); + } + } + + private static void IntIsOverZero(string variable, int? value, List errors) + { + if (value == null || value.Value < 1) + { + errors.Add($"{variable} is must be set and must be greater than 0."); + } + } + + private static void StringIsSet(string variable, string value, List errors) + { + if (string.IsNullOrWhiteSpace(value)) + { + errors.Add($"{variable} is must be set."); + } + } + + private static void Print(string variable, string value) + { + Console.WriteLine($"\t{variable}: '{value}'"); + } + + private static void Print(string variable, int? value) + { + if (value != null) Print(variable, value.ToString()!); + else Print(variable, ""); + } + } +} diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index a08476f..64a445f 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -1,7 +1,46 @@ -public class Program +using CodexNetDeployer; +using DistTestCore.Codex; +using DistTestCore.Marketplace; + +public class Program { public static void Main(string[] args) { - Console.WriteLine("Hello, World!"); + var nl = Environment.NewLine; + Console.WriteLine("CodexNetDeployer" + nl + nl); + + var argOrVar = new ArgOrVar(args); + + if (args.Any(a => a == "-h" || a == "--help" || a == "-?")) + { + argOrVar.PrintHelp(); + return; + } + + var config = new Configuration( + codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage), + gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage), + contractsImage: argOrVar.Get(ArgOrVar.ContractsImage, CodexContractsContainerRecipe.DockerImage), + kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile), + kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace), + numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes), + storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota) + ); + + Console.WriteLine("Using:"); + config.PrintConfig(); + Console.WriteLine(nl); + + var errors = config.Validate(); + if (errors.Any()) + { + Console.WriteLine($"Configuration errors: ({errors.Count})"); + foreach ( var error in errors ) Console.WriteLine("\t" + error); + Console.WriteLine(nl); + argOrVar.PrintHelp(); + return; + } + + } } From ee5a46694040bb546b8bca9ac89271bf2f797fcf Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 10:17:12 +0200 Subject: [PATCH 09/23] Wiring up the starter class --- CodexNetDeployer/ArgOrVar.cs | 5 +++- CodexNetDeployer/Configuration.cs | 12 +++++++-- CodexNetDeployer/Deployer.cs | 41 +++++++++++++++++++++++++++++++ CodexNetDeployer/NullLog.cs | 41 +++++++++++++++++++++++++++++++ CodexNetDeployer/Program.cs | 15 ++++++++++- DistTestCore/Configuration.cs | 19 ++++++++------ Logging/BaseLog.cs | 12 ++++----- Utils/ParseEnum.cs | 10 ++++++++ 8 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 CodexNetDeployer/Deployer.cs create mode 100644 CodexNetDeployer/NullLog.cs create mode 100644 Utils/ParseEnum.cs diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs index df70171..90c34ae 100644 --- a/CodexNetDeployer/ArgOrVar.cs +++ b/CodexNetDeployer/ArgOrVar.cs @@ -9,6 +9,7 @@ public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment."); public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created."); public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node."); + public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]"); private readonly string[] args; @@ -53,11 +54,13 @@ "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); + Console.WriteLine("CodexNetDeployer assumes you are running this tool from *outside* the Kubernetes cluster you want to deploy to. " + + "If you are running this from a container inside the cluster, add the argument '--internal'." + nl); Console.Write("\t[ CLI argument ] or [ Environment variable ]"); Console.CursorLeft = 70; Console.Write("(Description)" + nl); - var fields = GetType().GetFields();// System.Reflection.BindingFlags.Public & System.Reflection.BindingFlags.Static); + var fields = GetType().GetFields(); foreach (var field in fields) { var value = (ArgVar)field.GetValue(null)!; diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 3abb7a4..0a3504c 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,4 +1,6 @@ -namespace CodexNetDeployer +using DistTestCore; + +namespace CodexNetDeployer { public class Configuration { @@ -9,7 +11,9 @@ string kubeConfigFile, string kubeNamespace, int? numberOfCodexNodes, - int? storageQuota) + int? storageQuota, + string codexLogLevel, + TestRunnerLocation runnerLocation) { CodexImage = codexImage; GethImage = gethImage; @@ -18,6 +22,8 @@ KubeNamespace = kubeNamespace; NumberOfCodexNodes = numberOfCodexNodes; StorageQuota = storageQuota; + CodexLogLevel = codexLogLevel; + RunnerLocation = runnerLocation; } public string CodexImage { get; } @@ -27,6 +33,8 @@ public string KubeNamespace { get; } public int? NumberOfCodexNodes { get; } public int? StorageQuota { get; } + public string CodexLogLevel { get; } + public TestRunnerLocation RunnerLocation { get; } public void PrintConfig() { diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs new file mode 100644 index 0000000..0683e5c --- /dev/null +++ b/CodexNetDeployer/Deployer.cs @@ -0,0 +1,41 @@ +using DistTestCore; +using DistTestCore.Codex; +using Utils; + +namespace CodexNetDeployer +{ + public class Deployer + { + private readonly Configuration config; + + public Deployer(Configuration config) + { + this.config = config; + } + + public void Deploy() + { + var log = new NullLog(); + var lifecycleConfig = new DistTestCore.Configuration + ( + kubeConfigFile: config.KubeConfigFile, + logPath: "null", + logDebug: false, + dataFilesPath: "notUsed", + codexLogLevel: ParseEnum.Parse(config.CodexLogLevel), + runnerLocation: config.RunnerLocation + ); + + var timeset = new DefaultTimeSet(); + var kubeConfig = new KubernetesWorkflow.Configuration( + k8sNamespacePrefix: config.KubeNamespace, + kubeConfigFile: config.KubeConfigFile, + operationTimeout: timeset.K8sOperationTimeout(), + retryDelay: timeset.WaitForK8sServiceDelay()); + + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset); + var workflowCreator = new KubernetesWorkflow.WorkflowCreator(log, kubeConfig); + var starter = new CodexStarter(lifecycle, workflowCreator); + } + } +} diff --git a/CodexNetDeployer/NullLog.cs b/CodexNetDeployer/NullLog.cs new file mode 100644 index 0000000..c64190b --- /dev/null +++ b/CodexNetDeployer/NullLog.cs @@ -0,0 +1,41 @@ +using Logging; + +namespace CodexNetDeployer +{ + public class NullLog : TestLog + { + public NullLog() : base("NULL", false, "NULL") + { + } + + protected override LogFile CreateLogFile() + { + return null!; + } + + public override void Log(string message) + { + } + + public override void Debug(string message = "", int skipFrames = 0) + { + } + + public override void Error(string message) + { + Console.WriteLine("Error: " + message); + } + + public override void MarkAsFailed() + { + } + + public override void AddStringReplace(string from, string to) + { + } + + public override void Delete() + { + } + } +} diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 64a445f..582d2a1 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -1,6 +1,8 @@ using CodexNetDeployer; +using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; +using Configuration = CodexNetDeployer.Configuration; public class Program { @@ -17,6 +19,12 @@ public class Program return; } + var location = TestRunnerLocation.ExternalToCluster; + if (args.Any(a => a == "--internal")) + { + location = TestRunnerLocation.InternalToCluster; + } + var config = new Configuration( codexImage: argOrVar.Get(ArgOrVar.CodexImage, CodexContainerRecipe.DockerImage), gethImage: argOrVar.Get(ArgOrVar.GethImage, GethContainerRecipe.DockerImage), @@ -24,7 +32,9 @@ public class Program kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile), kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace), numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes), - storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota) + storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota), + codexLogLevel: argOrVar.Get(ArgOrVar.LogLevel), + runnerLocation: location ); Console.WriteLine("Using:"); @@ -41,6 +51,9 @@ public class Program return; } + var deployer = new Deployer(config); + deployer.Deploy(); + Console.WriteLine("Done!"); } } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index af76127..03514cd 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -19,8 +19,18 @@ namespace DistTestCore logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); - codexLogLevel = ParseEnum(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); - runnerLocation = ParseEnum(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); + codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); + runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); + } + + public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) + { + this.kubeConfigFile = kubeConfigFile; + this.logPath = logPath; + this.logDebug = logDebug; + this.dataFilesPath = dataFilesPath; + this.codexLogLevel = codexLogLevel; + this.runnerLocation = runnerLocation; } public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet) @@ -75,11 +85,6 @@ namespace DistTestCore if (v == null) return defaultValue; return v; } - - private static T ParseEnum(string value) - { - return (T)Enum.Parse(typeof(T), value, true); - } } public enum TestRunnerLocation diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs index 2c69f9c..c312256 100644 --- a/Logging/BaseLog.cs +++ b/Logging/BaseLog.cs @@ -25,12 +25,12 @@ namespace Logging } } - public void Log(string message) + public virtual void Log(string message) { LogFile.Write(ApplyReplacements(message)); } - public void Debug(string message = "", int skipFrames = 0) + public virtual void Debug(string message = "", int skipFrames = 0) { if (debug) { @@ -40,25 +40,25 @@ namespace Logging } } - public void Error(string message) + public virtual void Error(string message) { Log($"[ERROR] {message}"); } - public void MarkAsFailed() + public virtual void MarkAsFailed() { if (hasFailed) return; hasFailed = true; LogFile.ConcatToFilename("_FAILED"); } - public void AddStringReplace(string from, string to) + public virtual void AddStringReplace(string from, string to) { if (string.IsNullOrWhiteSpace(from)) return; replacements.Add(new BaseLogStringReplacement(from, to)); } - public void Delete() + public virtual void Delete() { File.Delete(LogFile.FullFilename); } diff --git a/Utils/ParseEnum.cs b/Utils/ParseEnum.cs new file mode 100644 index 0000000..3b3d0cb --- /dev/null +++ b/Utils/ParseEnum.cs @@ -0,0 +1,10 @@ +namespace Utils +{ + public static class ParseEnum + { + public static T Parse(string value) + { + return (T)Enum.Parse(typeof(T), value, true); + } + } +} From 7fdca9ffdb17e350d5362913131a5c041fb7afdf Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 10:33:21 +0200 Subject: [PATCH 10/23] all wired up --- CodexNetDeployer/Configuration.cs | 5 +++-- CodexNetDeployer/Deployer.cs | 30 ++++++++++++++++++++++++++---- CodexNetDeployer/Program.cs | 3 ++- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 0a3504c..3e5346c 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,4 +1,5 @@ using DistTestCore; +using DistTestCore.Codex; namespace CodexNetDeployer { @@ -12,7 +13,7 @@ namespace CodexNetDeployer string kubeNamespace, int? numberOfCodexNodes, int? storageQuota, - string codexLogLevel, + CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) { CodexImage = codexImage; @@ -33,7 +34,7 @@ namespace CodexNetDeployer public string KubeNamespace { get; } public int? NumberOfCodexNodes { get; } public int? StorageQuota { get; } - public string CodexLogLevel { get; } + public CodexLogLevel CodexLogLevel { get; } public TestRunnerLocation RunnerLocation { get; } public void PrintConfig() diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 0683e5c..03f428a 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -1,6 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; -using Utils; namespace CodexNetDeployer { @@ -14,6 +12,25 @@ namespace CodexNetDeployer } public void Deploy() + { + Log("Initializing..."); + var starter = CreateStarter(); + + Log("Preparing configuration..."); + var setup = new CodexSetup(config.NumberOfCodexNodes!.Value, config.CodexLogLevel); + + Log("Creating resources..."); + var group = (CodexNodeGroup) starter.BringOnline(setup); + + var containers = group.Containers; + foreach (var container in containers.Containers) + { + var pod = container.Pod.PodInfo; + Log($"Container '{container.Name}' online. Pod: '{pod.Name}@{pod.Ip}' on '{pod.K8SNodeName}'."); + } + } + + private CodexStarter CreateStarter() { var log = new NullLog(); var lifecycleConfig = new DistTestCore.Configuration @@ -22,7 +39,7 @@ namespace CodexNetDeployer logPath: "null", logDebug: false, dataFilesPath: "notUsed", - codexLogLevel: ParseEnum.Parse(config.CodexLogLevel), + codexLogLevel: config.CodexLogLevel, runnerLocation: config.RunnerLocation ); @@ -35,7 +52,12 @@ namespace CodexNetDeployer var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset); var workflowCreator = new KubernetesWorkflow.WorkflowCreator(log, kubeConfig); - var starter = new CodexStarter(lifecycle, workflowCreator); + return new CodexStarter(lifecycle, workflowCreator); + } + + private void Log(string msg) + { + Console.WriteLine(msg); } } } diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 582d2a1..6f61fc7 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -2,6 +2,7 @@ using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; +using Utils; using Configuration = CodexNetDeployer.Configuration; public class Program @@ -33,7 +34,7 @@ public class Program kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace), numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes), storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota), - codexLogLevel: argOrVar.Get(ArgOrVar.LogLevel), + codexLogLevel: ParseEnum.Parse(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))), runnerLocation: location ); From cd69565c2e62d28f6f45d7cb4a8b329e62a5093f Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 14:37:37 +0200 Subject: [PATCH 11/23] Successful local deploy --- CodexNetDeployer/Deployer.cs | 75 +++++++++++++++---- CodexNetDeployer/NullLog.cs | 2 + CodexNetDeployer/Program.cs | 8 ++ DistTestCore/Codex/CodexContainerRecipe.cs | 8 +- DistTestCore/Configuration.cs | 2 +- DistTestCore/GethStarter.cs | 2 + .../Marketplace/MarketplaceInitialConfig.cs | 1 + DistTestCore/TestLifecycle.cs | 7 +- 8 files changed, 85 insertions(+), 20 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 03f428a..40849b5 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -1,41 +1,84 @@ using DistTestCore; +using DistTestCore.Codex; +using DistTestCore.Marketplace; +using KubernetesWorkflow; +using System.ComponentModel; namespace CodexNetDeployer { public class Deployer { private readonly Configuration config; + private readonly NullLog log; + private readonly DefaultTimeSet timeset; public Deployer(Configuration config) { this.config = config; + log = new NullLog(); + timeset = new DefaultTimeSet(); } public void Deploy() { Log("Initializing..."); - var starter = CreateStarter(); + var (workflowCreator, lifecycle) = CreateFacilities(); Log("Preparing configuration..."); - var setup = new CodexSetup(config.NumberOfCodexNodes!.Value, config.CodexLogLevel); + // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. + var setup = new CodexSetup(999, config.CodexLogLevel); + setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens()); - Log("Creating resources..."); - var group = (CodexNodeGroup) starter.BringOnline(setup); + Log("Creating Geth instance and deploying contracts..."); + var gethStarter = new GethStarter(lifecycle, workflowCreator); + var gethResults = gethStarter.BringOnlineMarketplaceFor(setup); - var containers = group.Containers; - foreach (var container in containers.Containers) + Log("Geth started. Codex contracts deployed."); + + Log("Starting Codex nodes..."); + + // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. + var bootstrapSpr = ""; // The first one will be used to bootstrap the others. + for (var i = 0; i < config.NumberOfCodexNodes; i++) { - var pod = container.Pod.PodInfo; - Log($"Container '{container.Name}' online. Pod: '{pod.Name}@{pod.Ip}' on '{pod.K8SNodeName}'."); + Console.Write($" - {i} = "); + var workflow = workflowCreator.CreateWorkflow(); + var workflowStartup = new StartupConfig(); + var codexStart = new CodexStartupConfig(config.CodexLogLevel); + workflowStartup.Add(codexStart); + if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; + codexStart.StorageQuota = config.StorageQuota.Value.MB(); + var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens()); + marketplaceConfig.AccountIndexOverride = i; + codexStart.MarketplaceConfig = marketplaceConfig; + workflowStartup.Add(gethResults); + + var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); + + var container = containers.Containers.First(); + var address = lifecycle.Configuration.GetAddress(container); + var codexNode = new CodexNode(log, timeset, address); + var debugInfo = codexNode.GetDebugInfo(); + + if (!string.IsNullOrWhiteSpace(debugInfo.spr)) + { + var pod = container.Pod.PodInfo; + Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}'" + Environment.NewLine); + + if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; + } + else + { + Console.Write("Unknown failure." + Environment.NewLine); + } } } - private CodexStarter CreateStarter() + private (WorkflowCreator, TestLifecycle) CreateFacilities() { - var log = new NullLog(); var lifecycleConfig = new DistTestCore.Configuration ( - kubeConfigFile: config.KubeConfigFile, + kubeConfigFile: null, //config.KubeConfigFile, logPath: "null", logDebug: false, dataFilesPath: "notUsed", @@ -43,16 +86,16 @@ namespace CodexNetDeployer runnerLocation: config.RunnerLocation ); - var timeset = new DefaultTimeSet(); var kubeConfig = new KubernetesWorkflow.Configuration( k8sNamespacePrefix: config.KubeNamespace, - kubeConfigFile: config.KubeConfigFile, + kubeConfigFile: null, // config.KubeConfigFile, operationTimeout: timeset.K8sOperationTimeout(), retryDelay: timeset.WaitForK8sServiceDelay()); - var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset); - var workflowCreator = new KubernetesWorkflow.WorkflowCreator(log, kubeConfig); - return new CodexStarter(lifecycle, workflowCreator); + var workflowCreator = new WorkflowCreator(log, kubeConfig); + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); + + return (workflowCreator, lifecycle); } private void Log(string msg) diff --git a/CodexNetDeployer/NullLog.cs b/CodexNetDeployer/NullLog.cs index c64190b..8417d39 100644 --- a/CodexNetDeployer/NullLog.cs +++ b/CodexNetDeployer/NullLog.cs @@ -15,10 +15,12 @@ namespace CodexNetDeployer public override void Log(string message) { + //Console.WriteLine(message); } public override void Debug(string message = "", int skipFrames = 0) { + //Console.WriteLine(message); } public override void Error(string message) diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 6f61fc7..ce59a6d 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -9,6 +9,14 @@ public class Program { public static void Main(string[] args) { + args = new[] + { + @"--kube-config=C:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml", + "--kube-namespace=testing-deployer", + "--nodes=3", + "--storage-quota=1024" + }; + var nl = Environment.NewLine; Console.WriteLine("CodexNetDeployer" + nl + nl); diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index f327927..4cf4be1 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -50,7 +50,7 @@ namespace DistTestCore.Codex { var gethConfig = startupConfig.Get(); var companionNode = gethConfig.CompanionNode; - var companionNodeAccount = companionNode.Accounts[Index]; + var companionNodeAccount = companionNode.Accounts[GetAccountIndex(config.MarketplaceConfig)]; Additional(companionNodeAccount); var ip = companionNode.RunningContainer.Pod.PodInfo.Ip; @@ -62,5 +62,11 @@ namespace DistTestCore.Codex AddEnvVar("PERSISTENCE", "1"); } } + + private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig) + { + if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value; + return Index; + } } } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 03514cd..fceb4e0 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -17,7 +17,7 @@ namespace DistTestCore { kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null); logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); - logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; + logDebug = GetEnvVarOrDefault("LOGDEBUG", "true").ToLowerInvariant() == "true"; dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); codexLogLevel = ParseEnum.Parse(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); runnerLocation = ParseEnum.Parse(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 92af53f..3b9a9b1 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -33,6 +33,8 @@ namespace DistTestCore private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode) { + if (marketplaceConfig.InitialTestTokens.Amount == 0) return; + var interaction = marketplaceNetwork.StartInteraction(lifecycle); var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; diff --git a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs index 1b66199..b802f33 100644 --- a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -10,5 +10,6 @@ public Ether InitialEth { get; } public TestToken InitialTestTokens { get; } + public int? AccountIndexOverride { get; set; } } } diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index ffd34b1..667b96c 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -7,15 +7,18 @@ namespace DistTestCore { public class TestLifecycle { - private readonly WorkflowCreator workflowCreator; private DateTime testStart = DateTime.MinValue; public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet) + : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet))) + { + } + + public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) { Log = log; Configuration = configuration; TimeSet = timeSet; - workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)); FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this, workflowCreator); From f242ad5a16645bc8a746acb32ec634f4af6d46b2 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 22 Jun 2023 15:58:18 +0200 Subject: [PATCH 12/23] Adds option to declare validators. --- CodexNetDeployer/ArgOrVar.cs | 1 + CodexNetDeployer/Configuration.cs | 8 ++++++++ CodexNetDeployer/Deployer.cs | 5 +++-- CodexNetDeployer/Program.cs | 4 +++- DistTestCore/Codex/CodexContainerRecipe.cs | 5 +++++ DistTestCore/CodexSetup.cs | 8 +++++++- DistTestCore/Marketplace/MarketplaceInitialConfig.cs | 4 +++- 7 files changed, 30 insertions(+), 5 deletions(-) diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs index 90c34ae..f4e35cd 100644 --- a/CodexNetDeployer/ArgOrVar.cs +++ b/CodexNetDeployer/ArgOrVar.cs @@ -8,6 +8,7 @@ public static readonly ArgVar KubeConfigFile = new ArgVar("kube-config", "KUBECONFIG", "Path to Kubeconfig file."); public static readonly ArgVar KubeNamespace = new ArgVar("kube-namespace", "KUBENAMESPACE", "Kubernetes namespace to be used for deployment."); public static readonly ArgVar NumberOfCodexNodes = new ArgVar("nodes", "NODES", "Number of Codex nodes to be created."); + public static readonly ArgVar NumberOfValidatorNodes = new ArgVar("validators", "VALIDATORS", "Number of Codex nodes that will be validating."); public static readonly ArgVar StorageQuota = new ArgVar("storage-quota", "STORAGEQUOTA", "Storage quota in megabytes used by each Codex node."); public static readonly ArgVar LogLevel = new ArgVar("log-level", "LOGLEVEL", "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]"); diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 3e5346c..2383047 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -12,6 +12,7 @@ namespace CodexNetDeployer string kubeConfigFile, string kubeNamespace, int? numberOfCodexNodes, + int? numberOfValidators, int? storageQuota, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation) @@ -22,6 +23,7 @@ namespace CodexNetDeployer KubeConfigFile = kubeConfigFile; KubeNamespace = kubeNamespace; NumberOfCodexNodes = numberOfCodexNodes; + NumberOfValidators = numberOfValidators; StorageQuota = storageQuota; CodexLogLevel = codexLogLevel; RunnerLocation = runnerLocation; @@ -33,6 +35,7 @@ namespace CodexNetDeployer public string KubeConfigFile { get; } public string KubeNamespace { get; } public int? NumberOfCodexNodes { get; } + public int? NumberOfValidators { get; } public int? StorageQuota { get; } public CodexLogLevel CodexLogLevel { get; } public TestRunnerLocation RunnerLocation { get; } @@ -50,6 +53,11 @@ namespace CodexNetDeployer onString: (n, v) => StringIsSet(n, v, errors), onInt: (n, v) => IntIsOverZero(n, v, errors)); + if (NumberOfValidators > NumberOfCodexNodes) + { + errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes})."); + } + return errors; } diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 40849b5..a4298b9 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -39,16 +39,16 @@ namespace CodexNetDeployer // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. var bootstrapSpr = ""; // The first one will be used to bootstrap the others. + int validatorsLeft = config.NumberOfValidators!.Value; for (var i = 0; i < config.NumberOfCodexNodes; i++) { Console.Write($" - {i} = "); var workflow = workflowCreator.CreateWorkflow(); var workflowStartup = new StartupConfig(); var codexStart = new CodexStartupConfig(config.CodexLogLevel); - workflowStartup.Add(codexStart); if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; codexStart.StorageQuota = config.StorageQuota.Value.MB(); - var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens()); + var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; workflowStartup.Add(gethResults); @@ -66,6 +66,7 @@ namespace CodexNetDeployer Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}'" + Environment.NewLine); if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; + validatorsLeft--; } else { diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index ce59a6d..eb3efc3 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -13,7 +13,8 @@ public class Program { @"--kube-config=C:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml", "--kube-namespace=testing-deployer", - "--nodes=3", + "--nodes=5", + "--validators=3", "--storage-quota=1024" }; @@ -41,6 +42,7 @@ public class Program kubeConfigFile: argOrVar.Get(ArgOrVar.KubeConfigFile), kubeNamespace: argOrVar.Get(ArgOrVar.KubeNamespace), numberOfCodexNodes: argOrVar.GetInt(ArgOrVar.NumberOfCodexNodes), + numberOfValidators: argOrVar.GetInt(ArgOrVar.NumberOfValidatorNodes), storageQuota: argOrVar.GetInt(ArgOrVar.StorageQuota), codexLogLevel: ParseEnum.Parse(argOrVar.Get(ArgOrVar.LogLevel, nameof(CodexLogLevel.Debug))), runnerLocation: location diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 4cf4be1..b7c3b94 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -60,6 +60,11 @@ namespace DistTestCore.Codex AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account); AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address); AddEnvVar("PERSISTENCE", "1"); + + if (config.MarketplaceConfig.IsValidator) + { + AddEnvVar("VALIDATOR", "1"); + } } } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 83c5b9b..8c1a73c 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -13,6 +13,7 @@ namespace DistTestCore ICodexSetup EnableMetrics(); ICodexSetup EnableMarketplace(TestToken initialBalance); ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther); + ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator); } public class CodexSetup : CodexStartupConfig, ICodexSetup @@ -62,7 +63,12 @@ namespace DistTestCore public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther) { - MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance); + return EnableMarketplace(initialBalance, initialEther, false); + } + + public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator) + { + MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance, isValidator); return this; } diff --git a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs index b802f33..c51d79f 100644 --- a/DistTestCore/Marketplace/MarketplaceInitialConfig.cs +++ b/DistTestCore/Marketplace/MarketplaceInitialConfig.cs @@ -2,14 +2,16 @@ { public class MarketplaceInitialConfig { - public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens) + public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens, bool isValidator) { InitialEth = initialEth; InitialTestTokens = initialTestTokens; + IsValidator = isValidator; } public Ether InitialEth { get; } public TestToken InitialTestTokens { get; } + public bool IsValidator { get; } public int? AccountIndexOverride { get; set; } } } From 4f0a278df1d838a0b9c26cd24c5f4e9ba9def47c Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 08:18:48 +0200 Subject: [PATCH 13/23] Flips internal/external default. --- CodexNetDeployer/ArgOrVar.cs | 4 ++-- CodexNetDeployer/Deployer.cs | 6 ++++-- CodexNetDeployer/Program.cs | 9 +++++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/CodexNetDeployer/ArgOrVar.cs b/CodexNetDeployer/ArgOrVar.cs index f4e35cd..5844b68 100644 --- a/CodexNetDeployer/ArgOrVar.cs +++ b/CodexNetDeployer/ArgOrVar.cs @@ -55,8 +55,8 @@ "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); - Console.WriteLine("CodexNetDeployer assumes you are running this tool from *outside* the Kubernetes cluster you want to deploy to. " + - "If you are running this from a container inside the cluster, add the argument '--internal'." + nl); + Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " + + "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); Console.Write("\t[ CLI argument ] or [ Environment variable ]"); Console.CursorLeft = 70; diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index a4298b9..137ce88 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -46,12 +46,14 @@ namespace CodexNetDeployer var workflow = workflowCreator.CreateWorkflow(); var workflowStartup = new StartupConfig(); var codexStart = new CodexStartupConfig(config.CodexLogLevel); + workflowStartup.Add(gethResults); + workflowStartup.Add(codexStart); + if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; codexStart.StorageQuota = config.StorageQuota.Value.MB(); var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; - workflowStartup.Add(gethResults); var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); @@ -63,7 +65,7 @@ namespace CodexNetDeployer if (!string.IsNullOrWhiteSpace(debugInfo.spr)) { var pod = container.Pod.PodInfo; - Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}'" + Environment.NewLine); + Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine); if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; validatorsLeft--; diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index eb3efc3..805b654 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -15,7 +15,8 @@ public class Program "--kube-namespace=testing-deployer", "--nodes=5", "--validators=3", - "--storage-quota=1024" + "--storage-quota=1024", + "--external" }; var nl = Environment.NewLine; @@ -29,10 +30,10 @@ public class Program return; } - var location = TestRunnerLocation.ExternalToCluster; - if (args.Any(a => a == "--internal")) + var location = TestRunnerLocation.InternalToCluster; + if (args.Any(a => a == "--external")) { - location = TestRunnerLocation.InternalToCluster; + location = TestRunnerLocation.ExternalToCluster; } var config = new Configuration( From aa9b667940291bd8d1966eea2f149c1fa1b2567b Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 08:44:27 +0200 Subject: [PATCH 14/23] Writes codex-deployment.json which can be used by the test running. --- CodexNetDeployer/CodexNodeStarter.cs | 75 +++++++++++++++++++++++++++ CodexNetDeployer/Deployer.cs | 45 +++------------- CodexNetDeployer/Program.cs | 7 ++- DistTestCore/Codex/CodexDeployment.cs | 17 ++++++ 4 files changed, 106 insertions(+), 38 deletions(-) create mode 100644 CodexNetDeployer/CodexNodeStarter.cs create mode 100644 DistTestCore/Codex/CodexDeployment.cs diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs new file mode 100644 index 0000000..ee071db --- /dev/null +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -0,0 +1,75 @@ +using DistTestCore; +using DistTestCore.Codex; +using DistTestCore.Marketplace; +using KubernetesWorkflow; +using Logging; + +namespace CodexNetDeployer +{ + public class CodexNodeStarter + { + private readonly Configuration config; + private readonly WorkflowCreator workflowCreator; + private readonly TestLifecycle lifecycle; + private readonly BaseLog log; + private readonly ITimeSet timeSet; + private readonly GethStartResult gethResult; + private string bootstrapSpr = ""; + private int validatorsLeft; + + public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, BaseLog log, ITimeSet timeSet, GethStartResult gethResult, int numberOfValidators) + { + this.config = config; + this.workflowCreator = workflowCreator; + this.lifecycle = lifecycle; + this.log = log; + this.timeSet = timeSet; + this.gethResult = gethResult; + this.validatorsLeft = numberOfValidators; + } + + public RunningContainer? Start(int i) + { + Console.Write($" - {i} = "); + var workflow = workflowCreator.CreateWorkflow(); + var workflowStartup = new StartupConfig(); + workflowStartup.Add(gethResult); + workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft)); + + var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); + + var container = containers.Containers.First(); + var address = lifecycle.Configuration.GetAddress(container); + var codexNode = new CodexNode(log, timeSet, address); + var debugInfo = codexNode.GetDebugInfo(); + + if (!string.IsNullOrWhiteSpace(debugInfo.spr)) + { + var pod = container.Pod.PodInfo; + Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine); + + if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; + validatorsLeft--; + return container; + } + else + { + Console.Write("Unknown failure." + Environment.NewLine); + return null; + } + } + + private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft) + { + var codexStart = new CodexStartupConfig(config.CodexLogLevel); + + if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; + codexStart.StorageQuota = config.StorageQuota!.Value.MB(); + var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); + marketplaceConfig.AccountIndexOverride = i; + codexStart.MarketplaceConfig = marketplaceConfig; + + return codexStart; + } + } +} diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 137ce88..b6a1df2 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -1,8 +1,6 @@ using DistTestCore; using DistTestCore.Codex; -using DistTestCore.Marketplace; using KubernetesWorkflow; -using System.ComponentModel; namespace CodexNetDeployer { @@ -19,7 +17,7 @@ namespace CodexNetDeployer timeset = new DefaultTimeSet(); } - public void Deploy() + public CodexDeployment Deploy() { Log("Initializing..."); var (workflowCreator, lifecycle) = CreateFacilities(); @@ -34,47 +32,20 @@ namespace CodexNetDeployer var gethResults = gethStarter.BringOnlineMarketplaceFor(setup); Log("Geth started. Codex contracts deployed."); + Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts."); Log("Starting Codex nodes..."); // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. - var bootstrapSpr = ""; // The first one will be used to bootstrap the others. - int validatorsLeft = config.NumberOfValidators!.Value; + var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, log, timeset, gethResults, config.NumberOfValidators!.Value); + var codexContainers = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { - Console.Write($" - {i} = "); - var workflow = workflowCreator.CreateWorkflow(); - var workflowStartup = new StartupConfig(); - var codexStart = new CodexStartupConfig(config.CodexLogLevel); - workflowStartup.Add(gethResults); - workflowStartup.Add(codexStart); - - if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr; - codexStart.StorageQuota = config.StorageQuota.Value.MB(); - var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); - marketplaceConfig.AccountIndexOverride = i; - codexStart.MarketplaceConfig = marketplaceConfig; - - var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup); - - var container = containers.Containers.First(); - var address = lifecycle.Configuration.GetAddress(container); - var codexNode = new CodexNode(log, timeset, address); - var debugInfo = codexNode.GetDebugInfo(); - - if (!string.IsNullOrWhiteSpace(debugInfo.spr)) - { - var pod = container.Pod.PodInfo; - Console.Write($"Online ({pod.Name} at {pod.Ip} on '{pod.K8SNodeName}')" + Environment.NewLine); - - if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; - validatorsLeft--; - } - else - { - Console.Write("Unknown failure." + Environment.NewLine); - } + var container = codexStarter.Start(i); + if (container != null) codexContainers.Add(container); } + + return new CodexDeployment(gethResults, codexContainers.ToArray()); } private (WorkflowCreator, TestLifecycle) CreateFacilities() diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 805b654..8c29f3d 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -2,6 +2,7 @@ using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; +using Newtonsoft.Json; using Utils; using Configuration = CodexNetDeployer.Configuration; @@ -64,7 +65,11 @@ public class Program } var deployer = new Deployer(config); - deployer.Deploy(); + var deployment = deployer.Deploy(); + + Console.WriteLine("Writing codex-deployment.json..."); + + File.WriteAllText("codex-deployment.json", JsonConvert.SerializeObject(deployment, Formatting.Indented)); Console.WriteLine("Done!"); } diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs new file mode 100644 index 0000000..f86ad31 --- /dev/null +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -0,0 +1,17 @@ +using DistTestCore.Marketplace; +using KubernetesWorkflow; + +namespace DistTestCore.Codex +{ + public class CodexDeployment + { + public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers) + { + GethStartResult = gethStartResult; + CodexContainers = codexContainers; + } + + public GethStartResult GethStartResult { get; } + public RunningContainer[] CodexContainers { get; } + } +} From acfd1f1f101a2c8d8d845d75bad49f56c0e4ee01 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 08:45:16 +0200 Subject: [PATCH 15/23] Removes local debug help --- CodexNetDeployer/Deployer.cs | 4 ++-- CodexNetDeployer/Program.cs | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index b6a1df2..a5aa56b 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -52,7 +52,7 @@ namespace CodexNetDeployer { var lifecycleConfig = new DistTestCore.Configuration ( - kubeConfigFile: null, //config.KubeConfigFile, + kubeConfigFile: config.KubeConfigFile, logPath: "null", logDebug: false, dataFilesPath: "notUsed", @@ -62,7 +62,7 @@ namespace CodexNetDeployer var kubeConfig = new KubernetesWorkflow.Configuration( k8sNamespacePrefix: config.KubeNamespace, - kubeConfigFile: null, // config.KubeConfigFile, + kubeConfigFile: config.KubeConfigFile, operationTimeout: timeset.K8sOperationTimeout(), retryDelay: timeset.WaitForK8sServiceDelay()); diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index 8c29f3d..462667c 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -10,16 +10,6 @@ public class Program { public static void Main(string[] args) { - args = new[] - { - @"--kube-config=C:\Users\Ben\.kube\codex-tests-ams3-dev-kubeconfig.yaml", - "--kube-namespace=testing-deployer", - "--nodes=5", - "--validators=3", - "--storage-quota=1024", - "--external" - }; - var nl = Environment.NewLine; Console.WriteLine("CodexNetDeployer" + nl + nl); From d9665f3e793fae1f111085abae7843e364261e9e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 08:57:31 +0200 Subject: [PATCH 16/23] Successful remote test deploy. Adds option to remove random guid from k8s namespace. --- CodexNetDeployer/Deployer.cs | 2 +- KubernetesWorkflow/WorkflowCreator.cs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index a5aa56b..2870e4d 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -66,7 +66,7 @@ namespace CodexNetDeployer operationTimeout: timeset.K8sOperationTimeout(), retryDelay: timeset.WaitForK8sServiceDelay()); - var workflowCreator = new WorkflowCreator(log, kubeConfig); + var workflowCreator = new WorkflowCreator(log, kubeConfig, testNamespacePostfix: string.Empty); var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator); return (workflowCreator, lifecycle); diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index d03dca7..ea1775a 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -13,10 +13,15 @@ namespace KubernetesWorkflow private readonly string testNamespace; public WorkflowCreator(BaseLog log, Configuration configuration) + : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant()) + { + } + + public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix) { cluster = new K8sCluster(configuration); this.log = log; - testNamespace = Guid.NewGuid().ToString().ToLowerInvariant(); + testNamespace = testNamespacePostfix; } public StartupWorkflow CreateWorkflow() From 2e09c9135e460c461208fa91ee21723dd2ce2345 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 09:08:18 +0200 Subject: [PATCH 17/23] Adds metadata about codex deployment --- CodexNetDeployer/Deployer.cs | 15 +++++++++++++- DistTestCore/Codex/CodexDeployment.cs | 30 ++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 2870e4d..3e476b4 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -45,7 +45,7 @@ namespace CodexNetDeployer if (container != null) codexContainers.Add(container); } - return new CodexDeployment(gethResults, codexContainers.ToArray()); + return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata()); } private (WorkflowCreator, TestLifecycle) CreateFacilities() @@ -72,6 +72,19 @@ namespace CodexNetDeployer return (workflowCreator, lifecycle); } + private DeploymentMetadata CreateMetadata() + { + return new DeploymentMetadata( + codexImage: config.CodexImage, + gethImage: config.GethImage, + contractsImage: config.ContractsImage, + kubeNamespace: config.KubeNamespace, + numberOfCodexNodes: config.NumberOfCodexNodes!.Value, + numberOfValidators: config.NumberOfValidators!.Value, + storageQuotaMB: config.StorageQuota!.Value, + codexLogLevel: config.CodexLogLevel); + } + private void Log(string msg) { Console.WriteLine(msg); diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs index f86ad31..cc4246e 100644 --- a/DistTestCore/Codex/CodexDeployment.cs +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -5,13 +5,41 @@ namespace DistTestCore.Codex { public class CodexDeployment { - public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers) + public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, DeploymentMetadata metadata) { GethStartResult = gethStartResult; CodexContainers = codexContainers; + Metadata = metadata; } public GethStartResult GethStartResult { get; } public RunningContainer[] CodexContainers { get; } + public DeploymentMetadata Metadata { get; } + } + + public class DeploymentMetadata + { + public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel) + { + DeployDateTimeUtc = DateTime.UtcNow; + CodexImage = codexImage; + GethImage = gethImage; + ContractsImage = contractsImage; + KubeNamespace = kubeNamespace; + NumberOfCodexNodes = numberOfCodexNodes; + NumberOfValidators = numberOfValidators; + StorageQuotaMB = storageQuotaMB; + CodexLogLevel = codexLogLevel; + } + + public string CodexImage { get; } + public DateTime DeployDateTimeUtc { get; } + public string GethImage { get; } + public string ContractsImage { get; } + public string KubeNamespace { get; } + public int NumberOfCodexNodes { get; } + public int NumberOfValidators { get; } + public int StorageQuotaMB { get; } + public CodexLogLevel CodexLogLevel { get; } } } From 091eae36cc8b70dc004f1bf58732845d5a6574b0 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 10:14:16 +0200 Subject: [PATCH 18/23] Wires in codex-deployment json. --- ContinuousTests/CodexNodeFactory.cs | 11 ++---- ContinuousTests/Configuration.cs | 50 ++++++++++++++----------- ContinuousTests/ContinuousTestRunner.cs | 16 ++------ ContinuousTests/SingleTestRun.cs | 15 ++++---- 4 files changed, 44 insertions(+), 48 deletions(-) diff --git a/ContinuousTests/CodexNodeFactory.cs b/ContinuousTests/CodexNodeFactory.cs index a3bf54b..6992398 100644 --- a/ContinuousTests/CodexNodeFactory.cs +++ b/ContinuousTests/CodexNodeFactory.cs @@ -1,20 +1,17 @@ using DistTestCore; using DistTestCore.Codex; +using KubernetesWorkflow; using Logging; -using Utils; namespace ContinuousTests { public class CodexNodeFactory { - public CodexNode[] Create(string[] urls, BaseLog log, ITimeSet timeSet) + public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet) { - return urls.Select(url => + return containers.Select(container => { - var cutIndex = url.LastIndexOf(':'); - var host = url.Substring(0, cutIndex); - var port = url.Substring(cutIndex + 1); - var address = new Address(host, Convert.ToInt32(port)); + var address = container.ClusterInternalAddress; return new CodexNode(log, timeSet, address); }).ToArray(); } diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index 4e6278e..f61c463 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -1,11 +1,12 @@ -using Newtonsoft.Json; +using DistTestCore.Codex; +using Newtonsoft.Json; namespace ContinuousTests { public class Configuration { public string LogPath { get; set; } = string.Empty; - public string[] CodexUrls { get; set; } = Array.Empty(); + public CodexDeployment CodexDeployment { get; set; } = null!; public int SleepSecondsPerSingleTest { get; set; } public int SleepSecondsPerAllTests { get; set; } public bool KeepPassedTestLogs { get; set; } @@ -37,32 +38,30 @@ namespace ContinuousTests } var logPath = Environment.GetEnvironmentVariable("LOGPATH"); - var codexUrls = Environment.GetEnvironmentVariable("CODEXURLS"); + var codexDeploymentJson = Environment.GetEnvironmentVariable("CODEXDEPLOYMENT"); var sleepPerSingle = Environment.GetEnvironmentVariable("SLEEPSECONDSPERSINGLETEST"); var sleepPerAll = Environment.GetEnvironmentVariable("SLEEPSECONDSPERALLTESTS"); var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS"); if (!string.IsNullOrEmpty(logPath) && - !string.IsNullOrEmpty(codexUrls) && + !string.IsNullOrEmpty(codexDeploymentJson) && !string.IsNullOrEmpty(sleepPerSingle) && !string.IsNullOrEmpty(sleepPerAll)) { - var urls = codexUrls.Split(';', StringSplitOptions.RemoveEmptyEntries); - int secondsSingle; - int secondsAll; - if (int.TryParse(sleepPerSingle, out secondsSingle) && int.TryParse(sleepPerAll, out secondsAll)) + try { - if (urls.Length > 0) - { - return new Configuration - { - LogPath = logPath, - CodexUrls = urls, - SleepSecondsPerSingleTest = secondsSingle, - SleepSecondsPerAllTests = secondsAll, - KeepPassedTestLogs = keep == "1" - }; - } + return new Configuration + { + LogPath = logPath, + CodexDeployment = ParseCodexDeploymentJson(codexDeploymentJson), + SleepSecondsPerSingleTest = Convert.ToInt32(sleepPerSingle), + SleepSecondsPerAllTests = Convert.ToInt32(sleepPerAll), + KeepPassedTestLogs = keep == "1" + }; + } + catch (Exception ex) + { + Console.WriteLine("Exception: " + ex); } } @@ -70,7 +69,7 @@ namespace ContinuousTests throw new Exception($"Unable to load configuration from '{filename}', and " + "unable to load configuration from environment variables. " + nl + "'LOGPATH' = Path where log files will be saved." + nl + - "'CODEXURLS' = Semi-colon separated URLs to codex APIs. e.g. 'https://hostaddr_one:port;https://hostaddr_two:port'" + nl + + "'CODEXDEPLOYMENT' = Path to codex-deployment JSON file." + nl + "'SLEEPSECONDSPERSINGLETEST' = Seconds to sleep after each individual test." + nl + "'SLEEPSECONDSPERALLTESTS' = Seconds to sleep after all tests, before starting again." + nl + "'KEEPPASSEDTESTLOGS' = (Optional, default: 0) Set to '1' to keep log files of tests that passed." + nl + @@ -95,10 +94,17 @@ namespace ContinuousTests throw new Exception($"Unvalid logpath set: '{configuration.LogPath}'"); } - if (!configuration.CodexUrls.Any()) + if (configuration.CodexDeployment != null && configuration.CodexDeployment.CodexContainers.Any()) { - throw new Exception("No Codex URLs found."); + throw new Exception("No Codex deployment found."); } } + + private CodexDeployment ParseCodexDeploymentJson(string filename) + { + var d = JsonConvert.DeserializeObject(File.ReadAllText(filename))!; + if (d == null) throw new Exception("Unable to parse " + filename); + return d; + } } } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index cb6bc3f..117cedc 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -12,15 +12,7 @@ namespace ContinuousTests public void Run() { - var config = //configLoader.Load(); - new Configuration - { - CodexUrls =new[] { "http://localhost:8080", "http://localhost:8081" }, - LogPath = "logs", - KeepPassedTestLogs = false, - SleepSecondsPerAllTests = 1, - SleepSecondsPerSingleTest = 1, - }; + var config = configLoader.Load(); StartupChecks(config); while (true) @@ -73,9 +65,9 @@ namespace ContinuousTests var errors = new List(); foreach (var test in tests) { - if (test.RequiredNumberOfNodes > config.CodexUrls.Length) + if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length) { - errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Configuration only has {config.CodexUrls.Length}"); + errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}"); } } @@ -92,7 +84,7 @@ namespace ContinuousTests private void CheckCodexNodes(BaseLog log, Configuration config) { - var nodes = codexNodeFactory.Create(config.CodexUrls, log, new DefaultTimeSet()); + var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet()); var pass = true; foreach (var n in nodes) { diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 978f4bd..6782424 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -2,6 +2,7 @@ using DistTestCore; using Logging; using Utils; +using KubernetesWorkflow; namespace ContinuousTests { @@ -37,18 +38,18 @@ namespace ContinuousTests private CodexNode[] CreateRandomNodes(int number, BaseLog testLog) { - var urls = SelectRandomUrls(number); - testLog.Log("Selected nodes: " + string.Join(",", urls)); - return codexNodeFactory.Create(urls, testLog, test.TimeSet); + var containers = SelectRandomContainers(number); + testLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name))); + return codexNodeFactory.Create(containers, testLog, test.TimeSet); } - private string[] SelectRandomUrls(int number) + private RunningContainer[] SelectRandomContainers(int number) { - var urls = config.CodexUrls.ToList(); - var result = new string[number]; + var containers = config.CodexDeployment.CodexContainers.ToList(); + var result = new RunningContainer[number]; for (var i = 0; i < number; i++) { - result[i] = urls.PickOneRandom(); + result[i] = containers.PickOneRandom(); } return result; } From 9d3874c88f87000cc7debd0bd6739b401304f0f0 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 10:35:23 +0200 Subject: [PATCH 19/23] Runner correctly ingests codex-deployment.json --- ContinuousTests/Configuration.cs | 2 +- DistTestCore/Marketplace/GethStartResult.cs | 5 ++++- KubernetesWorkflow/RunningContainers.cs | 16 ++-------------- KubernetesWorkflow/StartupWorkflow.cs | 17 ++++++++++++++++- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index f61c463..e654893 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -94,7 +94,7 @@ namespace ContinuousTests throw new Exception($"Unvalid logpath set: '{configuration.LogPath}'"); } - if (configuration.CodexDeployment != null && configuration.CodexDeployment.CodexContainers.Any()) + if (configuration.CodexDeployment == null || !configuration.CodexDeployment.CodexContainers.Any()) { throw new Exception("No Codex deployment found."); } diff --git a/DistTestCore/Marketplace/GethStartResult.cs b/DistTestCore/Marketplace/GethStartResult.cs index 412d288..0ba1e58 100644 --- a/DistTestCore/Marketplace/GethStartResult.cs +++ b/DistTestCore/Marketplace/GethStartResult.cs @@ -1,4 +1,6 @@ -namespace DistTestCore.Marketplace +using Newtonsoft.Json; + +namespace DistTestCore.Marketplace { public class GethStartResult { @@ -9,6 +11,7 @@ CompanionNode = companionNode; } + [JsonIgnore] public IMarketplaceAccessFactory MarketplaceAccessFactory { get; } public MarketplaceNetwork MarketplaceNetwork { get; } public GethCompanionNodeInfo CompanionNode { get; } diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 6e2224d..bbfe360 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -23,12 +23,12 @@ namespace KubernetesWorkflow public class RunningContainer { - public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, Address clusterExternalAddress, Address clusterInternalAddress) + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress) { Pod = pod; Recipe = recipe; ServicePorts = servicePorts; - Name = GetContainerName(recipe, startupConfig); + Name = name; ClusterExternalAddress = clusterExternalAddress; ClusterInternalAddress = clusterInternalAddress; } @@ -39,17 +39,5 @@ namespace KubernetesWorkflow public Port[] ServicePorts { get; } public Address ClusterExternalAddress { get; } public Address ClusterInternalAddress { get; } - - private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig) - { - if (!string.IsNullOrEmpty(startupConfig.NameOverride)) - { - return $"<{startupConfig.NameOverride}{recipe.Number}>"; - } - else - { - return $"<{recipe.Name}>"; - } - } } } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 8a223cf..c7e3bfb 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -81,13 +81,28 @@ namespace KubernetesWorkflow var servicePorts = runningPod.GetServicePortsForContainerRecipe(r); log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}"); - return new RunningContainer(runningPod, r, servicePorts, startupConfig, + var name = GetContainerName(r, startupConfig); + + return new RunningContainer(runningPod, r, servicePorts, name, GetContainerExternalAddress(runningPod, servicePorts), GetContainerInternalAddress(r)); }).ToArray(); } + private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig) + { + if (startupConfig == null) return ""; + if (!string.IsNullOrEmpty(startupConfig.NameOverride)) + { + return $"<{startupConfig.NameOverride}{recipe.Number}>"; + } + else + { + return $"<{recipe.Name}>"; + } + } + private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts) { return new Address( From c08507e1b854c3303b21559f7cd43ab2c7619be1 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 23 Jun 2023 11:38:30 +0200 Subject: [PATCH 20/23] wip: figuring out log handling for continous tests --- ContinuousTests/AllTestsRun.cs | 8 +-- ContinuousTests/ContinuousTest.cs | 5 ++ ContinuousTests/ContinuousTestRunner.cs | 27 ++++++---- ContinuousTests/TestHandle.cs | 69 ++++++++++++++++++++++++ ContinuousTests/TestMomentAttribute.cs | 12 +++++ ContinuousTests/Tests/MarketplaceTest.cs | 28 ++++++++++ 6 files changed, 137 insertions(+), 12 deletions(-) create mode 100644 ContinuousTests/TestHandle.cs create mode 100644 ContinuousTests/TestMomentAttribute.cs create mode 100644 ContinuousTests/Tests/MarketplaceTest.cs diff --git a/ContinuousTests/AllTestsRun.cs b/ContinuousTests/AllTestsRun.cs index d825cbe..965523a 100644 --- a/ContinuousTests/AllTestsRun.cs +++ b/ContinuousTests/AllTestsRun.cs @@ -18,11 +18,13 @@ namespace ContinuousTests public ContinuousTestResult RunAll() { - var remainingTests = testFinder.CreateTests().ToList(); + var tests = testFinder.CreateTests().ToList(); + var handles = tests.Select(t => new TestHandle(t)).ToArray(); + var result = ContinuousTestResult.Passed; - while (remainingTests.Any()) + while (tests.Any()) { - var test = remainingTests.PickOneRandom(); + var test = tests.PickOneRandom(); var testLog = log.CreateTestLog(test.Name); var singleTestRun = new SingleTestRun(config, test, testLog); diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 725f456..fcdaecb 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -11,6 +11,11 @@ namespace ContinuousTests public abstract class ContinuousTest { + protected const int Zero = 0; + protected const int HourOne = 3600; + protected const int DayOne = HourOne * 24; + protected const int DayThree = DayOne * 3; + private const string UploadFailedMessage = "Unable to store block"; public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager) diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 117cedc..2c3fe82 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -4,7 +4,13 @@ using Logging; namespace ContinuousTests { - public class ContinuousTestRunner + public interface ITestResultHandler + { + void TestPassed(ContinuousTest test); + void TestFailed(ContinuousTest test); + } + + public class ContinuousTestRunner : ITestResultHandler { private readonly ConfigLoader configLoader = new ConfigLoader(); private readonly TestFactory testFactory = new TestFactory(); @@ -18,12 +24,11 @@ namespace ContinuousTests while (true) { var log = new FixtureLog(new LogConfig(config.LogPath, false), "ContinuousTestsRun"); - var allTestsRun = new AllTestsRun(config, log, testFactory); + var allTestsRun = new AllTestsRun(config, log, testFactory, this); - var result = ContinuousTestResult.Passed; try { - result = allTestsRun.RunAll(); + allTestsRun.RunAll(); } catch (Exception ex) { @@ -119,11 +124,15 @@ namespace ContinuousTests } return true; } - } - public enum ContinuousTestResult - { - Passed, - Failed + public void TestPassed(ContinuousTest test) + { + throw new NotImplementedException(); + } + + public void TestFailed(ContinuousTest test) + { + throw new NotImplementedException(); + } } } diff --git a/ContinuousTests/TestHandle.cs b/ContinuousTests/TestHandle.cs new file mode 100644 index 0000000..8bbc28b --- /dev/null +++ b/ContinuousTests/TestHandle.cs @@ -0,0 +1,69 @@ +using System.Reflection; + +namespace ContinuousTests +{ + public class TestHandle + { + private readonly List moments = new List(); + + public TestHandle(ContinuousTest test) + { + Test = test; + + ReflectTestMoments(); + + if (!moments.Any()) throw new Exception("Test has no moments."); + if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception("Test has duplicate moments"); + } + + public ContinuousTest Test { get; } + + public int? GetNextMoment(int currentMoment) + { + var remainingMoments = moments.Where(m => m.Moment >= currentMoment).ToArray(); + if (!remainingMoments.Any()) return null; + return remainingMoments.Min(m => m.Moment); + } + + public int GetLastMoment() + { + return moments.Max(m => m.Moment); + } + + public void InvokeMoment(int currentMoment) + { + var moment = moments.SingleOrDefault(m => m.Moment == currentMoment); + if (moment == null) return; + + moment.Method.Invoke(Test, Array.Empty()); + } + + private void ReflectTestMoments() + { + var methods = Test.GetType().GetMethods() + .Where(m => m.GetCustomAttributes(typeof(TestMomentAttribute), false).Length > 0) + .ToArray(); + + foreach (var method in methods) + { + var moment = method.GetCustomAttribute(); + if (moment != null && moment.T >= 0) + { + moments.Add(new MethodMoment(method, moment.T)); + } + } + } + } + + public class MethodMoment + { + public MethodMoment(MethodInfo method, int moment) + { + Method = method; + Moment = moment; + } + + public MethodInfo Method { get; } + public int Moment { get; } + } +} diff --git a/ContinuousTests/TestMomentAttribute.cs b/ContinuousTests/TestMomentAttribute.cs new file mode 100644 index 0000000..991725e --- /dev/null +++ b/ContinuousTests/TestMomentAttribute.cs @@ -0,0 +1,12 @@ +namespace ContinuousTests +{ + public class TestMomentAttribute : Attribute + { + public TestMomentAttribute(int t) + { + T = t; + } + + public int T { get; } + } +} diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs new file mode 100644 index 0000000..c5fff94 --- /dev/null +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -0,0 +1,28 @@ +using DistTestCore.Codex; + +namespace ContinuousTests.Tests +{ + public class MarketplaceTest : ContinuousTest + { + public override int RequiredNumberOfNodes => 1; + + public override void Run() + { + } + + [TestMoment(t: Zero)] + public void NodePostsStorageRequest() + { + //var c = new KubernetesWorkflow.WorkflowCreator(Log, new KubernetesWorkflow.Configuration()); + //var flow = c.CreateWorkflow(); + //var rc = flow.Start(10, KubernetesWorkflow.Location.Unspecified, new CodexContainerRecipe(), new KubernetesWorkflow.StartupConfig()); + } + + [TestMoment(t: DayThree)] + public void NodeDownloadsStorageRequestData() + { + + } + } + +} From 9a9c740e1c8bd520ffd4acb24113657198ad3681 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 25 Jun 2023 09:53:10 +0200 Subject: [PATCH 21/23] Sets up new test execution loop --- ContinuousTests/AllTestsRun.cs | 51 -------- ContinuousTests/CodexNodeFactory.cs | 2 +- ContinuousTests/Configuration.cs | 37 +++--- ContinuousTests/ContinuousTest.cs | 10 +- ContinuousTests/ContinuousTestRunner.cs | 143 +++------------------- ContinuousTests/Program.cs | 2 + ContinuousTests/SingleTestRun.cs | 106 ++++++++++++++-- ContinuousTests/StartupChecker.cs | 102 +++++++++++++++ ContinuousTests/TestHandle.cs | 33 +++-- ContinuousTests/TestStarter.cs | 36 ++++++ ContinuousTests/Tests/MarketplaceTest.cs | 6 +- ContinuousTests/Tests/PerformanceTests.cs | 114 ++++++++--------- ContinuousTests/Tests/TwoClientTest.cs | 24 ++-- 13 files changed, 373 insertions(+), 293 deletions(-) delete mode 100644 ContinuousTests/AllTestsRun.cs create mode 100644 ContinuousTests/StartupChecker.cs create mode 100644 ContinuousTests/TestStarter.cs diff --git a/ContinuousTests/AllTestsRun.cs b/ContinuousTests/AllTestsRun.cs deleted file mode 100644 index 965523a..0000000 --- a/ContinuousTests/AllTestsRun.cs +++ /dev/null @@ -1,51 +0,0 @@ -using Logging; -using Utils; - -namespace ContinuousTests -{ - public class AllTestsRun - { - private readonly Configuration config; - private readonly FixtureLog log; - private readonly TestFactory testFinder; - - public AllTestsRun(Configuration config, FixtureLog log, TestFactory testFinder) - { - this.config = config; - this.log = log; - this.testFinder = testFinder; - } - - public ContinuousTestResult RunAll() - { - var tests = testFinder.CreateTests().ToList(); - var handles = tests.Select(t => new TestHandle(t)).ToArray(); - - var result = ContinuousTestResult.Passed; - while (tests.Any()) - { - var test = tests.PickOneRandom(); - var testLog = log.CreateTestLog(test.Name); - var singleTestRun = new SingleTestRun(config, test, testLog); - - log.Log($"Start '{test.Name}'"); - try - { - singleTestRun.Run(); - log.Log($"'{test.Name}' = Passed"); - if (!config.KeepPassedTestLogs) testLog.Delete(); - } - catch - { - log.Log($"'{test.Name}' = Failed"); - testLog.MarkAsFailed(); - result = ContinuousTestResult.Failed; - } - - Thread.Sleep(config.SleepSecondsPerSingleTest * 1000); - } - - return result; - } - } -} diff --git a/ContinuousTests/CodexNodeFactory.cs b/ContinuousTests/CodexNodeFactory.cs index 6992398..43b1fea 100644 --- a/ContinuousTests/CodexNodeFactory.cs +++ b/ContinuousTests/CodexNodeFactory.cs @@ -11,7 +11,7 @@ namespace ContinuousTests { return containers.Select(container => { - var address = container.ClusterInternalAddress; + var address = container.ClusterExternalAddress; return new CodexNode(log, timeSet, address); }).ToArray(); } diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index e654893..b8b5223 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -6,9 +6,8 @@ namespace ContinuousTests public class Configuration { public string LogPath { get; set; } = string.Empty; + public string DataPath { get; set; } = string.Empty; public CodexDeployment CodexDeployment { get; set; } = null!; - public int SleepSecondsPerSingleTest { get; set; } - public int SleepSecondsPerAllTests { get; set; } public bool KeepPassedTestLogs { get; set; } } @@ -37,11 +36,11 @@ namespace ContinuousTests catch { } } - var logPath = Environment.GetEnvironmentVariable("LOGPATH"); - var codexDeploymentJson = Environment.GetEnvironmentVariable("CODEXDEPLOYMENT"); - var sleepPerSingle = Environment.GetEnvironmentVariable("SLEEPSECONDSPERSINGLETEST"); - var sleepPerAll = Environment.GetEnvironmentVariable("SLEEPSECONDSPERALLTESTS"); - var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS"); + var logPath = "logs";// Environment.GetEnvironmentVariable("LOGPATH"); + var codexDeploymentJson = "C:\\Users\\Ben\\Desktop\\codex-deployment.json"; //Environment.GetEnvironmentVariable("CODEXDEPLOYMENT"); + var sleepPerSingle = "10";// Environment.GetEnvironmentVariable("SLEEPSECONDSPERSINGLETEST"); + var sleepPerAll = "10";// Environment.GetEnvironmentVariable("SLEEPSECONDSPERALLTESTS"); + var keep = ""; // Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS"); if (!string.IsNullOrEmpty(logPath) && !string.IsNullOrEmpty(codexDeploymentJson) && @@ -54,8 +53,8 @@ namespace ContinuousTests { LogPath = logPath, CodexDeployment = ParseCodexDeploymentJson(codexDeploymentJson), - SleepSecondsPerSingleTest = Convert.ToInt32(sleepPerSingle), - SleepSecondsPerAllTests = Convert.ToInt32(sleepPerAll), + //SleepSecondsPerSingleTest = Convert.ToInt32(sleepPerSingle), + //SleepSecondsPerAllTests = Convert.ToInt32(sleepPerAll), KeepPassedTestLogs = keep == "1" }; } @@ -78,16 +77,16 @@ namespace ContinuousTests private void Validate(Configuration configuration) { - if (configuration.SleepSecondsPerSingleTest < 1) - { - Console.WriteLine("Warning: configuration.SleepSecondsPerSingleTest was less than 1 seconds. Using 1 seconds instead!"); - configuration.SleepSecondsPerSingleTest = 1; - } - if (configuration.SleepSecondsPerAllTests < 1) - { - Console.WriteLine("Warning: configuration.SleepSecondsPerAllTests was less than 10 seconds. Using 10 seconds instead!"); - configuration.SleepSecondsPerAllTests = 10; - } + //if (configuration.SleepSecondsPerSingleTest < 1) + //{ + // Console.WriteLine("Warning: configuration.SleepSecondsPerSingleTest was less than 1 seconds. Using 1 seconds instead!"); + // configuration.SleepSecondsPerSingleTest = 1; + //} + //if (configuration.SleepSecondsPerAllTests < 1) + //{ + // Console.WriteLine("Warning: configuration.SleepSecondsPerAllTests was less than 10 seconds. Using 10 seconds instead!"); + // configuration.SleepSecondsPerAllTests = 10; + //} if (string.IsNullOrEmpty(configuration.LogPath)) { diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index fcdaecb..4df6dae 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -31,6 +31,8 @@ namespace ContinuousTests public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } } public abstract int RequiredNumberOfNodes { get; } + public abstract TimeSpan RunTestEvery { get; } + public abstract TestFailMode TestFailMode { get; } public string Name { @@ -40,8 +42,6 @@ namespace ContinuousTests } } - public abstract void Run(); - public ContentId? UploadFile(CodexNode node, TestFile file) { using var fileStream = File.OpenRead(file.Filename); @@ -84,4 +84,10 @@ namespace ContinuousTests } } } + + public enum TestFailMode + { + StopAfterFirstFailure, + AlwaysRunAllMoments + } } diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 2c3fe82..b1915e5 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -1,138 +1,31 @@ -using DistTestCore; -using DistTestCore.Codex; -using Logging; - -namespace ContinuousTests +namespace ContinuousTests { - public interface ITestResultHandler - { - void TestPassed(ContinuousTest test); - void TestFailed(ContinuousTest test); - } - - public class ContinuousTestRunner : ITestResultHandler + public class ContinuousTestRunner { private readonly ConfigLoader configLoader = new ConfigLoader(); private readonly TestFactory testFactory = new TestFactory(); - private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); + private readonly Configuration config; + private readonly StartupChecker startupChecker; + + public ContinuousTestRunner() + { + config = configLoader.Load(); + startupChecker = new StartupChecker(config); + } public void Run() { - var config = configLoader.Load(); - StartupChecks(config); + startupChecker.Check(); - while (true) + var allTests = testFactory.CreateTests(); + var testStarters = allTests.Select(t => new TestStarter(config, t.GetType(), t.RunTestEvery)).ToArray(); + + foreach (var t in testStarters) { - var log = new FixtureLog(new LogConfig(config.LogPath, false), "ContinuousTestsRun"); - var allTestsRun = new AllTestsRun(config, log, testFactory, this); - - try - { - allTestsRun.RunAll(); - } - catch (Exception ex) - { - log.Error($"Exception during test run: " + ex); - } - - if (result == ContinuousTestResult.Failed) - { - log.MarkAsFailed(); - } - if (!config.KeepPassedTestLogs && result == ContinuousTestResult.Passed) - { - log.DeleteFolder(); - } - - Thread.Sleep(config.SleepSecondsPerSingleTest * 1000); + t.Begin(); } - } - - private void StartupChecks(Configuration config) - { - var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks"); - log.Log("Starting continuous test run..."); - log.Log("Checking configuration..."); - PreflightCheck(config); - log.Log("Contacting Codex nodes..."); - CheckCodexNodes(log, config); - log.Log("All OK."); - } - - private void PreflightCheck(Configuration config) - { - var tests = testFactory.CreateTests(); - if (!tests.Any()) - { - throw new Exception("Unable to find any tests."); - } - - var errors = new List(); - foreach (var test in tests) - { - if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length) - { - errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}"); - } - } - - if (!Directory.Exists(config.LogPath)) - { - Directory.CreateDirectory(config.LogPath); - } - - if (errors.Any()) - { - throw new Exception("Prerun check failed: " + string.Join(", ", errors)); - } - } - - private void CheckCodexNodes(BaseLog log, Configuration config) - { - var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet()); - var pass = true; - foreach (var n in nodes) - { - log.Log($"Checking '{n.Address.Host}'..."); - - if (EnsureOnline(n)) - { - log.Log("OK"); - } - else - { - log.Error($"No response from '{n.Address.Host}'."); - pass = false; - } - } - if (!pass) - { - throw new Exception("Not all codex nodes responded."); - } - } - - private bool EnsureOnline(CodexNode n) - { - try - { - var info = n.GetDebugInfo(); - if (info == null || string.IsNullOrEmpty(info.id)) return false; - } - catch - { - return false; - } - return true; - } - - public void TestPassed(ContinuousTest test) - { - throw new NotImplementedException(); - } - - public void TestFailed(ContinuousTest test) - { - throw new NotImplementedException(); + + Thread.Sleep(TimeSpan.MaxValue); } } } diff --git a/ContinuousTests/Program.cs b/ContinuousTests/Program.cs index d3fcb1d..c265c36 100644 --- a/ContinuousTests/Program.cs +++ b/ContinuousTests/Program.cs @@ -4,6 +4,8 @@ public class Program { public static void Main(string[] args) { + Console.WriteLine("Codex Continous-Test-Runner."); + Console.WriteLine("Running..."); var runner = new ContinuousTestRunner(); runner.Run(); } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 6782424..f7740fe 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -9,38 +9,112 @@ namespace ContinuousTests public class SingleTestRun { private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); + private readonly List exceptions = new List(); private readonly Configuration config; - private readonly ContinuousTest test; + private readonly TestHandle handle; private readonly CodexNode[] nodes; private readonly FileManager fileManager; + private readonly FixtureLog fixtureLog; - public SingleTestRun(Configuration config, ContinuousTest test, BaseLog testLog) + public SingleTestRun(Configuration config, TestHandle handle) { this.config = config; - this.test = test; + this.handle = handle; - nodes = CreateRandomNodes(test.RequiredNumberOfNodes, testLog); - fileManager = new FileManager(testLog, new DistTestCore.Configuration()); + var testName = handle.Test.GetType().Name; + fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName); - test.Initialize(nodes, testLog, fileManager); + nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes); + fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration()); } public void Run() { - test.Run(); + Task.Run(() => + { + try + { + RunTest(); + + if (!config.KeepPassedTestLogs) fixtureLog.Delete(); + } + catch (Exception ex) + { + fixtureLog.Error("Test run failed with exception: " + ex); + fixtureLog.MarkAsFailed(); + } + fileManager.DeleteAllTestFiles(); + }); } - public void TearDown() + private void RunTest() { - test.Initialize(null!, null!, null!); - fileManager.DeleteAllTestFiles(); + var earliestMoment = handle.GetEarliestMoment(); + var lastMoment = handle.GetLastMoment(); + + var t = earliestMoment; + while (t <= lastMoment) + { + RunMoment(t); + + if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any()) + { + Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping..."); + throw exceptions.Single(); + } + + var nextMoment = handle.GetNextMoment(t); + if (nextMoment != null) + { + Log($" > Next TestMoment in {nextMoment.Value} seconds..."); + t += nextMoment.Value; + Thread.Sleep(nextMoment.Value * 1000); + } + else + { + Log(" > Completed last test moment. Test ended."); + } + } + + if (exceptions.Any()) throw exceptions.First(); } - private CodexNode[] CreateRandomNodes(int number, BaseLog testLog) + private void RunMoment(int t) + { + try + { + handle.InvokeMoment(t, InitializeTest); + } + catch (Exception ex) + { + Log($" > TestMoment yielded exception: " + ex); + exceptions.Add(ex); + } + + DecommissionTest(); + } + + private void InitializeTest(string name) + { + Log($" > Running TestMoment '{name}'"); + handle.Test.Initialize(nodes, fixtureLog, fileManager); + } + + private void DecommissionTest() + { + handle.Test.Initialize(null!, null!, null!); + } + + private void Log(string msg) + { + fixtureLog.Log(msg); + } + + private CodexNode[] CreateRandomNodes(int number) { var containers = SelectRandomContainers(number); - testLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name))); - return codexNodeFactory.Create(containers, testLog, test.TimeSet); + fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name))); + return codexNodeFactory.Create(containers, fixtureLog, handle.Test.TimeSet); } private RunningContainer[] SelectRandomContainers(int number) @@ -53,5 +127,11 @@ namespace ContinuousTests } return result; } + + private DistTestCore.Configuration CreateFileManagerConfiguration() + { + return new DistTestCore.Configuration(null, string.Empty, false, config.DataPath + Guid.NewGuid(), + CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster); + } } } diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs new file mode 100644 index 0000000..8000b5b --- /dev/null +++ b/ContinuousTests/StartupChecker.cs @@ -0,0 +1,102 @@ +using DistTestCore.Codex; +using DistTestCore; +using Logging; + +namespace ContinuousTests +{ + public class StartupChecker + { + private readonly TestFactory testFactory = new TestFactory(); + private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); + private readonly Configuration config; + + public StartupChecker(Configuration config) + { + this.config = config; + } + + public void Check() + { + var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks"); + log.Log("Starting continuous test run..."); + log.Log("Checking configuration..."); + PreflightCheck(config); + log.Log("Contacting Codex nodes..."); + CheckCodexNodes(log, config); + log.Log("All OK."); + } + + private void PreflightCheck(Configuration config) + { + var tests = testFactory.CreateTests(); + if (!tests.Any()) + { + throw new Exception("Unable to find any tests."); + } + foreach (var test in tests) + { + var handle = new TestHandle(test); + handle.GetEarliestMoment(); + handle.GetLastMoment(); + } + + var errors = new List(); + foreach (var test in tests) + { + if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length) + { + errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}"); + } + } + + if (!Directory.Exists(config.LogPath)) + { + Directory.CreateDirectory(config.LogPath); + } + + if (errors.Any()) + { + throw new Exception("Prerun check failed: " + string.Join(", ", errors)); + } + } + + private void CheckCodexNodes(BaseLog log, Configuration config) + { + var nodes = codexNodeFactory.Create(config.CodexDeployment.CodexContainers, log, new DefaultTimeSet()); + var pass = true; + foreach (var n in nodes) + { + log.Log($"Checking '{n.Address.Host}'..."); + + if (EnsureOnline(n)) + { + log.Log("OK"); + } + else + { + log.Error($"No response from '{n.Address.Host}'."); + pass = false; + } + } + if (!pass) + { + throw new Exception("Not all codex nodes responded."); + } + } + + private bool EnsureOnline(CodexNode n) + { + try + { + var info = n.GetDebugInfo(); + if (info == null || string.IsNullOrEmpty(info.id)) return false; + } + catch + { + return false; + } + return true; + } + + } +} diff --git a/ContinuousTests/TestHandle.cs b/ContinuousTests/TestHandle.cs index 8bbc28b..6decdc0 100644 --- a/ContinuousTests/TestHandle.cs +++ b/ContinuousTests/TestHandle.cs @@ -12,12 +12,23 @@ namespace ContinuousTests ReflectTestMoments(); - if (!moments.Any()) throw new Exception("Test has no moments."); - if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception("Test has duplicate moments"); + var testName = test.GetType().Name; + if (!moments.Any()) throw new Exception($"Test '{testName}' has no moments."); + if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception($"Test '{testName}' has duplicate moments"); } public ContinuousTest Test { get; } + public int GetEarliestMoment() + { + return moments.Min(m => m.Moment); + } + + public int GetLastMoment() + { + return moments.Max(m => m.Moment); + } + public int? GetNextMoment(int currentMoment) { var remainingMoments = moments.Where(m => m.Moment >= currentMoment).ToArray(); @@ -25,17 +36,16 @@ namespace ContinuousTests return remainingMoments.Min(m => m.Moment); } - public int GetLastMoment() - { - return moments.Max(m => m.Moment); - } - - public void InvokeMoment(int currentMoment) + public void InvokeMoment(int currentMoment, Action beforeInvoke) { var moment = moments.SingleOrDefault(m => m.Moment == currentMoment); if (moment == null) return; - moment.Method.Invoke(Test, Array.Empty()); + lock (MomentLock.Lock) + { + beforeInvoke(moment.Method.Name); + moment.Method.Invoke(Test, Array.Empty()); + } } private void ReflectTestMoments() @@ -66,4 +76,9 @@ namespace ContinuousTests public MethodInfo Method { get; } public int Moment { get; } } + + public static class MomentLock + { + public static readonly object Lock = new(); + } } diff --git a/ContinuousTests/TestStarter.cs b/ContinuousTests/TestStarter.cs new file mode 100644 index 0000000..68deb9a --- /dev/null +++ b/ContinuousTests/TestStarter.cs @@ -0,0 +1,36 @@ +namespace ContinuousTests +{ + public class TestStarter + { + private readonly Configuration config; + private readonly Type testType; + private readonly TimeSpan runsEvery; + + public TestStarter(Configuration config, Type testType, TimeSpan runsEvery) + { + this.config = config; + this.testType = testType; + this.runsEvery = runsEvery; + } + + public void Begin() + { + Task.Run(() => + { + while (true) + { + StartTest(); + Thread.Sleep(runsEvery); + } + }); + } + + private void StartTest() + { + var test = (ContinuousTest)Activator.CreateInstance(testType)!; + var handle = new TestHandle(test); + var run = new SingleTestRun(config, handle); + run.Run(); + } + } +} diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index c5fff94..e82e1f7 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -5,10 +5,8 @@ namespace ContinuousTests.Tests public class MarketplaceTest : ContinuousTest { public override int RequiredNumberOfNodes => 1; - - public override void Run() - { - } + public override TimeSpan RunTestEvery => TimeSpan.FromDays(1); + public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; [TestMoment(t: Zero)] public void NodePostsStorageRequest() diff --git a/ContinuousTests/Tests/PerformanceTests.cs b/ContinuousTests/Tests/PerformanceTests.cs index f0ee853..ce0af10 100644 --- a/ContinuousTests/Tests/PerformanceTests.cs +++ b/ContinuousTests/Tests/PerformanceTests.cs @@ -4,77 +4,77 @@ using NUnit.Framework; namespace ContinuousTests.Tests { - public class UploadPerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 1; + //public class UploadPerformanceTest : PerformanceTest + //{ + // public override int RequiredNumberOfNodes => 1; - public override void Run() - { - UploadTest(100, Nodes[0]); - } - } + // public override void Run() + // { + // UploadTest(100, Nodes[0]); + // } + //} - public class DownloadLocalPerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 1; + //public class DownloadLocalPerformanceTest : PerformanceTest + //{ + // public override int RequiredNumberOfNodes => 1; - public override void Run() - { - DownloadTest(100, Nodes[0], Nodes[0]); - } - } + // public override void Run() + // { + // DownloadTest(100, Nodes[0], Nodes[0]); + // } + //} - public class DownloadRemotePerformanceTest : PerformanceTest - { - public override int RequiredNumberOfNodes => 2; + //public class DownloadRemotePerformanceTest : PerformanceTest + //{ + // public override int RequiredNumberOfNodes => 2; - public override void Run() - { - DownloadTest(100, Nodes[0], Nodes[1]); - } - } + // public override void Run() + // { + // DownloadTest(100, Nodes[0], Nodes[1]); + // } + //} - public abstract class PerformanceTest : ContinuousTest - { - public void UploadTest(int megabytes, CodexNode uploadNode) - { - var file = FileManager.GenerateTestFile(megabytes.MB()); + //public abstract class PerformanceTest : ContinuousTest + //{ + // public void UploadTest(int megabytes, CodexNode uploadNode) + // { + // var file = FileManager.GenerateTestFile(megabytes.MB()); - var time = Measure(() => - { - UploadFile(uploadNode, file); - }); + // var time = Measure(() => + // { + // UploadFile(uploadNode, file); + // }); - var timePerMB = time / megabytes; + // var timePerMB = time / megabytes; - Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); - } + // Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); + // } - public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode) - { - var file = FileManager.GenerateTestFile(megabytes.MB()); + // public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode) + // { + // var file = FileManager.GenerateTestFile(megabytes.MB()); - var cid = UploadFile(uploadNode, file); - Assert.That(cid, Is.Not.Null); + // var cid = UploadFile(uploadNode, file); + // Assert.That(cid, Is.Not.Null); - TestFile? result = null; - var time = Measure(() => - { - result = DownloadContent(downloadNode, cid!); - }); + // TestFile? result = null; + // var time = Measure(() => + // { + // result = DownloadContent(downloadNode, cid!); + // }); - file.AssertIsEqual(result); + // file.AssertIsEqual(result); - var timePerMB = time / megabytes; + // var timePerMB = time / megabytes; - Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); - } + // Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); + // } - private static TimeSpan Measure(Action action) - { - var start = DateTime.UtcNow; - action(); - return DateTime.UtcNow - start; - } - } + // private static TimeSpan Measure(Action action) + // { + // var start = DateTime.UtcNow; + // action(); + // return DateTime.UtcNow - start; + // } + //} } diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index 07fa868..13bc631 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -3,20 +3,20 @@ using NUnit.Framework; namespace ContinuousTests.Tests { - public class TwoClientTest : ContinuousTest - { - public override int RequiredNumberOfNodes => 2; + //public class TwoClientTest : ContinuousTest + //{ + // public override int RequiredNumberOfNodes => 2; - public override void Run() - { - var file = FileManager.GenerateTestFile(10.MB()); + // public override void Run() + // { + // var file = FileManager.GenerateTestFile(10.MB()); - var cid = UploadFile(Nodes[0], file); - Assert.That(cid, Is.Not.Null); + // var cid = UploadFile(Nodes[0], file); + // Assert.That(cid, Is.Not.Null); - var dl = DownloadContent(Nodes[1], cid!); + // var dl = DownloadContent(Nodes[1], cid!); - file.AssertIsEqual(dl); - } - } + // file.AssertIsEqual(dl); + // } + //} } From 40b4d8aba3382fab553752be2140f7767a7ed18a Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 25 Jun 2023 10:50:01 +0200 Subject: [PATCH 22/23] Running two client test --- ContinuousTests/Configuration.cs | 39 +++++++++--------------- ContinuousTests/ContinuousTest.cs | 4 ++- ContinuousTests/ContinuousTestRunner.cs | 2 +- ContinuousTests/SingleTestRun.cs | 18 +++++++---- ContinuousTests/TestHandle.cs | 4 ++- ContinuousTests/Tests/MarketplaceTest.cs | 35 +++++++++++---------- ContinuousTests/Tests/TwoClientTest.cs | 34 +++++++++++++-------- 7 files changed, 72 insertions(+), 64 deletions(-) diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index b8b5223..3fdca13 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -36,25 +36,22 @@ namespace ContinuousTests catch { } } - var logPath = "logs";// Environment.GetEnvironmentVariable("LOGPATH"); - var codexDeploymentJson = "C:\\Users\\Ben\\Desktop\\codex-deployment.json"; //Environment.GetEnvironmentVariable("CODEXDEPLOYMENT"); - var sleepPerSingle = "10";// Environment.GetEnvironmentVariable("SLEEPSECONDSPERSINGLETEST"); - var sleepPerAll = "10";// Environment.GetEnvironmentVariable("SLEEPSECONDSPERALLTESTS"); - var keep = ""; // Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS"); + var logPath = Environment.GetEnvironmentVariable("LOGPATH"); + var dataPath = Environment.GetEnvironmentVariable("DATAPATH"); + var codexDeploymentJson = Environment.GetEnvironmentVariable("CODEXDEPLOYMENT"); + var keep = Environment.GetEnvironmentVariable("KEEPPASSEDTESTLOGS"); if (!string.IsNullOrEmpty(logPath) && - !string.IsNullOrEmpty(codexDeploymentJson) && - !string.IsNullOrEmpty(sleepPerSingle) && - !string.IsNullOrEmpty(sleepPerAll)) + !string.IsNullOrEmpty(dataPath) && + !string.IsNullOrEmpty(codexDeploymentJson)) { try { return new Configuration { LogPath = logPath, + DataPath = dataPath, CodexDeployment = ParseCodexDeploymentJson(codexDeploymentJson), - //SleepSecondsPerSingleTest = Convert.ToInt32(sleepPerSingle), - //SleepSecondsPerAllTests = Convert.ToInt32(sleepPerAll), KeepPassedTestLogs = keep == "1" }; } @@ -68,29 +65,21 @@ namespace ContinuousTests throw new Exception($"Unable to load configuration from '{filename}', and " + "unable to load configuration from environment variables. " + nl + "'LOGPATH' = Path where log files will be saved." + nl + + "'DATAPATH' = Path where temporary data files will be saved." + nl + "'CODEXDEPLOYMENT' = Path to codex-deployment JSON file." + nl + - "'SLEEPSECONDSPERSINGLETEST' = Seconds to sleep after each individual test." + nl + - "'SLEEPSECONDSPERALLTESTS' = Seconds to sleep after all tests, before starting again." + nl + - "'KEEPPASSEDTESTLOGS' = (Optional, default: 0) Set to '1' to keep log files of tests that passed." + nl + nl); } private void Validate(Configuration configuration) { - //if (configuration.SleepSecondsPerSingleTest < 1) - //{ - // Console.WriteLine("Warning: configuration.SleepSecondsPerSingleTest was less than 1 seconds. Using 1 seconds instead!"); - // configuration.SleepSecondsPerSingleTest = 1; - //} - //if (configuration.SleepSecondsPerAllTests < 1) - //{ - // Console.WriteLine("Warning: configuration.SleepSecondsPerAllTests was less than 10 seconds. Using 10 seconds instead!"); - // configuration.SleepSecondsPerAllTests = 10; - //} - if (string.IsNullOrEmpty(configuration.LogPath)) { - throw new Exception($"Unvalid logpath set: '{configuration.LogPath}'"); + throw new Exception($"Invalid LogPath set: '{configuration.LogPath}'"); + } + + if (string.IsNullOrEmpty(configuration.DataPath)) + { + throw new Exception($"Invalid DataPath set: '{configuration.DataPath}'"); } if (configuration.CodexDeployment == null || !configuration.CodexDeployment.CodexContainers.Any()) diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 4df6dae..c278aad 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -12,7 +12,9 @@ namespace ContinuousTests public abstract class ContinuousTest { protected const int Zero = 0; - protected const int HourOne = 3600; + protected const int MinuteOne = 60; + protected const int MinuteFive = MinuteOne * 5; + protected const int HourOne = MinuteOne * 60; protected const int DayOne = HourOne * 24; protected const int DayThree = DayOne * 3; diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index b1915e5..a2522aa 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -25,7 +25,7 @@ t.Begin(); } - Thread.Sleep(TimeSpan.MaxValue); + while (true) Thread.Sleep((2 ^ 31) - 1); } } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index f7740fe..8e384d3 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -15,6 +15,7 @@ namespace ContinuousTests private readonly CodexNode[] nodes; private readonly FileManager fileManager; private readonly FixtureLog fixtureLog; + private readonly string dataFolder; public SingleTestRun(Configuration config, TestHandle handle) { @@ -25,6 +26,7 @@ namespace ContinuousTests fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName); nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes); + dataFolder = config.DataPath + "-" + Guid.NewGuid(); fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration()); } @@ -44,16 +46,16 @@ namespace ContinuousTests fixtureLog.MarkAsFailed(); } fileManager.DeleteAllTestFiles(); + Directory.Delete(dataFolder, true); }); } private void RunTest() { var earliestMoment = handle.GetEarliestMoment(); - var lastMoment = handle.GetLastMoment(); var t = earliestMoment; - while (t <= lastMoment) + while (true) { RunMoment(t); @@ -72,11 +74,15 @@ namespace ContinuousTests } else { - Log(" > Completed last test moment. Test ended."); + if (exceptions.Any()) + { + Log(" > Completed last test moment. Test failed."); + throw exceptions.First(); + } + Log(" > Completed last test moment. Test passed."); + return; } } - - if (exceptions.Any()) throw exceptions.First(); } private void RunMoment(int t) @@ -130,7 +136,7 @@ namespace ContinuousTests private DistTestCore.Configuration CreateFileManagerConfiguration() { - return new DistTestCore.Configuration(null, string.Empty, false, config.DataPath + Guid.NewGuid(), + return new DistTestCore.Configuration(null, string.Empty, false, dataFolder, CodexLogLevel.Error, TestRunnerLocation.ExternalToCluster); } } diff --git a/ContinuousTests/TestHandle.cs b/ContinuousTests/TestHandle.cs index 6decdc0..7071d17 100644 --- a/ContinuousTests/TestHandle.cs +++ b/ContinuousTests/TestHandle.cs @@ -31,7 +31,7 @@ namespace ContinuousTests public int? GetNextMoment(int currentMoment) { - var remainingMoments = moments.Where(m => m.Moment >= currentMoment).ToArray(); + var remainingMoments = moments.Where(m => m.Moment > currentMoment).ToArray(); if (!remainingMoments.Any()) return null; return remainingMoments.Min(m => m.Moment); } @@ -71,6 +71,8 @@ namespace ContinuousTests { Method = method; Moment = moment; + + if (moment < 0) throw new Exception("Moment must be zero or greater."); } public MethodInfo Method { get; } diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index e82e1f7..03243c8 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -2,25 +2,24 @@ namespace ContinuousTests.Tests { - public class MarketplaceTest : ContinuousTest - { - public override int RequiredNumberOfNodes => 1; - public override TimeSpan RunTestEvery => TimeSpan.FromDays(1); - public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; + //public class MarketplaceTest : ContinuousTest + //{ + // public override int RequiredNumberOfNodes => 1; + // public override TimeSpan RunTestEvery => TimeSpan.FromDays(1); + // public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; - [TestMoment(t: Zero)] - public void NodePostsStorageRequest() - { - //var c = new KubernetesWorkflow.WorkflowCreator(Log, new KubernetesWorkflow.Configuration()); - //var flow = c.CreateWorkflow(); - //var rc = flow.Start(10, KubernetesWorkflow.Location.Unspecified, new CodexContainerRecipe(), new KubernetesWorkflow.StartupConfig()); - } + // [TestMoment(t: Zero)] + // public void NodePostsStorageRequest() + // { + // //var c = new KubernetesWorkflow.WorkflowCreator(Log, new KubernetesWorkflow.Configuration()); + // //var flow = c.CreateWorkflow(); + // //var rc = flow.Start(10, KubernetesWorkflow.Location.Unspecified, new CodexContainerRecipe(), new KubernetesWorkflow.StartupConfig()); + // } - [TestMoment(t: DayThree)] - public void NodeDownloadsStorageRequestData() - { - - } - } + // [TestMoment(t: DayThree)] + // public void NodeDownloadsStorageRequestData() + // { + // } + //} } diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index 13bc631..e6cbc1a 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -3,20 +3,30 @@ using NUnit.Framework; namespace ContinuousTests.Tests { - //public class TwoClientTest : ContinuousTest - //{ - // public override int RequiredNumberOfNodes => 2; + public class TwoClientTest : ContinuousTest + { + public override int RequiredNumberOfNodes => 2; + public override TimeSpan RunTestEvery => TimeSpan.FromHours(1); + public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure; - // public override void Run() - // { - // var file = FileManager.GenerateTestFile(10.MB()); + private ContentId? cid; + private TestFile file = null!; - // var cid = UploadFile(Nodes[0], file); - // Assert.That(cid, Is.Not.Null); + [TestMoment(t: Zero)] + public void UploadTestFile() + { + file = FileManager.GenerateTestFile(10.MB()); - // var dl = DownloadContent(Nodes[1], cid!); + cid = UploadFile(Nodes[0], file); + Assert.That(cid, Is.Not.Null); + } - // file.AssertIsEqual(dl); - // } - //} + [TestMoment(t: MinuteFive)] + public void DownloadTestFile() + { + var dl = DownloadContent(Nodes[1], cid!); + + file.AssertIsEqual(dl); + } + } } From ae3b4df92c9a1202d189e36e9009e61edb96e8ba Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 25 Jun 2023 10:55:55 +0200 Subject: [PATCH 23/23] Restores performance tests --- ContinuousTests/Tests/PerformanceTests.cs | 120 ++++++++++++---------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/ContinuousTests/Tests/PerformanceTests.cs b/ContinuousTests/Tests/PerformanceTests.cs index ce0af10..53af7be 100644 --- a/ContinuousTests/Tests/PerformanceTests.cs +++ b/ContinuousTests/Tests/PerformanceTests.cs @@ -4,77 +4,83 @@ using NUnit.Framework; namespace ContinuousTests.Tests { - //public class UploadPerformanceTest : PerformanceTest - //{ - // public override int RequiredNumberOfNodes => 1; + public class UploadPerformanceTest : PerformanceTest + { + public override int RequiredNumberOfNodes => 1; - // public override void Run() - // { - // UploadTest(100, Nodes[0]); - // } - //} + [TestMoment(t: Zero)] + public void UploadTest() + { + UploadTest(100, Nodes[0]); + } + } - //public class DownloadLocalPerformanceTest : PerformanceTest - //{ - // public override int RequiredNumberOfNodes => 1; + public class DownloadLocalPerformanceTest : PerformanceTest + { + public override int RequiredNumberOfNodes => 1; - // public override void Run() - // { - // DownloadTest(100, Nodes[0], Nodes[0]); - // } - //} + [TestMoment(t: Zero)] + public void DownloadTest() + { + DownloadTest(100, Nodes[0], Nodes[0]); + } + } - //public class DownloadRemotePerformanceTest : PerformanceTest - //{ - // public override int RequiredNumberOfNodes => 2; + public class DownloadRemotePerformanceTest : PerformanceTest + { + public override int RequiredNumberOfNodes => 2; - // public override void Run() - // { - // DownloadTest(100, Nodes[0], Nodes[1]); - // } - //} + [TestMoment(t: Zero)] + public void DownloadTest() + { + DownloadTest(100, Nodes[0], Nodes[1]); + } + } - //public abstract class PerformanceTest : ContinuousTest - //{ - // public void UploadTest(int megabytes, CodexNode uploadNode) - // { - // var file = FileManager.GenerateTestFile(megabytes.MB()); + public abstract class PerformanceTest : ContinuousTest + { + public override TimeSpan RunTestEvery => TimeSpan.FromHours(1); + public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; - // var time = Measure(() => - // { - // UploadFile(uploadNode, file); - // }); + public void UploadTest(int megabytes, CodexNode uploadNode) + { + var file = FileManager.GenerateTestFile(megabytes.MB()); - // var timePerMB = time / megabytes; + var time = Measure(() => + { + UploadFile(uploadNode, file); + }); - // Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); - // } + var timePerMB = time / megabytes; - // public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode) - // { - // var file = FileManager.GenerateTestFile(megabytes.MB()); + Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); + } - // var cid = UploadFile(uploadNode, file); - // Assert.That(cid, Is.Not.Null); + public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode) + { + var file = FileManager.GenerateTestFile(megabytes.MB()); - // TestFile? result = null; - // var time = Measure(() => - // { - // result = DownloadContent(downloadNode, cid!); - // }); + var cid = UploadFile(uploadNode, file); + Assert.That(cid, Is.Not.Null); - // file.AssertIsEqual(result); + TestFile? result = null; + var time = Measure(() => + { + result = DownloadContent(downloadNode, cid!); + }); - // var timePerMB = time / megabytes; + file.AssertIsEqual(result); - // Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); - // } + var timePerMB = time / megabytes; - // private static TimeSpan Measure(Action action) - // { - // var start = DateTime.UtcNow; - // action(); - // return DateTime.UtcNow - start; - // } - //} + Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); + } + + private static TimeSpan Measure(Action action) + { + var start = DateTime.UtcNow; + action(); + return DateTime.UtcNow - start; + } + } }