diff --git a/ContinuousTests/CodexNodeFactory.cs b/ContinuousTests/CodexNodeFactory.cs new file mode 100644 index 00000000..a3bf54b0 --- /dev/null +++ b/ContinuousTests/CodexNodeFactory.cs @@ -0,0 +1,22 @@ +using DistTestCore; +using DistTestCore.Codex; +using Logging; +using Utils; + +namespace ContinuousTests +{ + public class CodexNodeFactory + { + public CodexNode[] Create(string[] urls, BaseLog log, ITimeSet timeSet) + { + return urls.Select(url => + { + var cutIndex = url.LastIndexOf(':'); + var host = url.Substring(0, cutIndex); + var port = url.Substring(cutIndex + 1); + var address = new Address(host, Convert.ToInt32(port)); + return new CodexNode(log, timeSet, address); + }).ToArray(); + } + } +} diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index 5ccb8456..34421efc 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -40,7 +40,7 @@ namespace ContinuousTests if (!string.IsNullOrEmpty(logPath) && !string.IsNullOrEmpty(codexUrls) && !string.IsNullOrEmpty(sleep)) { var urls = codexUrls.Split(';', StringSplitOptions.RemoveEmptyEntries); - var ms = 0; + int ms; if (int.TryParse(sleep, out ms)) { if (urls.Length > 0) diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs new file mode 100644 index 00000000..725f4561 --- /dev/null +++ b/ContinuousTests/ContinuousTest.cs @@ -0,0 +1,82 @@ +using DistTestCore; +using DistTestCore.Codex; +using Logging; + +namespace ContinuousTests +{ + public abstract class ContinuousTestLongTimeouts : ContinuousTest + { + public override ITimeSet TimeSet => new LongTimeSet(); + } + + public abstract class ContinuousTest + { + private const string UploadFailedMessage = "Unable to store block"; + + public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager) + { + Nodes = nodes; + Log = log; + FileManager = fileManager; + } + + public CodexNode[] Nodes { get; private set; } = null!; + public BaseLog Log { get; private set; } = null!; + public IFileManager FileManager { get; private set; } = null!; + public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } } + + public abstract int RequiredNumberOfNodes { get; } + + public string Name + { + get + { + return GetType().Name; + } + } + + public abstract void Run(); + + public ContentId? UploadFile(CodexNode node, TestFile file) + { + using var fileStream = File.OpenRead(file.Filename); + + var logMessage = $"Uploading file {file.Describe()}..."; + var response = Stopwatch.Measure(Log, logMessage, () => + { + return node.UploadFile(fileStream); + }); + + if (response.StartsWith(UploadFailedMessage)) + { + return null; + } + Log.Log($"Uploaded file. Received contentId: '{response}'."); + return new ContentId(response); + } + + public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "") + { + var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; + var file = FileManager.CreateEmptyTestFile(fileLabel); + Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file)); + Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'."); + return file; + } + + private void DownloadToFile(CodexNode node, string contentId, TestFile file) + { + using var fileStream = File.OpenWrite(file.Filename); + try + { + using var downloadStream = node.DownloadFile(contentId); + downloadStream.CopyTo(fileStream); + } + catch + { + Log.Log($"Failed to download file '{contentId}'."); + throw; + } + } + } +} diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index caa9fbfe..639390c7 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -1,14 +1,14 @@ using DistTestCore; using DistTestCore.Codex; using Logging; -using Utils; namespace ContinuousTests { public class ContinuousTestRunner { private readonly ConfigLoader configLoader = new ConfigLoader(); - private readonly TestFinder testFinder = new TestFinder(); + private readonly TestFactory testFactory = new TestFactory(); + private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); public void Run() { @@ -19,13 +19,13 @@ namespace ContinuousTests log.Log("Checking configuration..."); PreflightCheck(config); log.Log("Contacting Codex nodes..."); - var nodes = CreateCodexNodes(log, new LongTimeSet(), config); - log.Log("OK"); + CheckCodexNodes(log, config); + log.Log("All OK."); log.Log(""); while (true) { - var run = new TestRun(config, log, testFinder, nodes); + var run = new TestRun(config, log, testFactory); try { @@ -42,7 +42,7 @@ namespace ContinuousTests private void PreflightCheck(Configuration config) { - var tests = testFinder.GetTests(); + var tests = testFactory.CreateTests(); if (!tests.Any()) { throw new Exception("Unable to find any tests."); @@ -57,20 +57,20 @@ namespace ContinuousTests } } + if (!Directory.Exists(config.LogPath)) + { + Directory.CreateDirectory(config.LogPath); + } + if (errors.Any()) { throw new Exception("Prerun check failed: " + string.Join(", ", errors)); } } - private CodexNode[] CreateCodexNodes(BaseLog log, ITimeSet timeSet, Configuration config) + private void CheckCodexNodes(BaseLog log, Configuration config) { - var nodes = config.CodexUrls.Select(url => - { - var address = new Address(url, 1234); - return new CodexNode(log, timeSet, address); - }).ToArray(); - + var nodes = codexNodeFactory.Create(config.CodexUrls, log, new DefaultTimeSet()); var pass = true; foreach (var n in nodes) { @@ -90,8 +90,6 @@ namespace ContinuousTests { throw new Exception("Not all codex nodes responded."); } - - return nodes; } private bool EnsureOnline(CodexNode n) diff --git a/ContinuousTests/IContinuousTest.cs b/ContinuousTests/IContinuousTest.cs deleted file mode 100644 index 0374ffd2..00000000 --- a/ContinuousTests/IContinuousTest.cs +++ /dev/null @@ -1,33 +0,0 @@ -using DistTestCore; -using DistTestCore.Codex; -using Logging; - -namespace ContinuousTests -{ - public interface IContinuousTest - { - string Name { get; } - int RequiredNumberOfNodes { get; } - - void Run(); - } - - public abstract class ContinuousTest : IContinuousTest - { - public CodexNode[] Nodes { get; set; } = null!; - public BaseLog Log { get; set; } = null!; - public FileManager FileManager { get; set; } = null!; - - public abstract int RequiredNumberOfNodes { get; } - - public string Name - { - get - { - return GetType().Name; - } - } - - public abstract void Run(); - } -} diff --git a/ContinuousTests/TestFactory.cs b/ContinuousTests/TestFactory.cs new file mode 100644 index 00000000..3a4992a9 --- /dev/null +++ b/ContinuousTests/TestFactory.cs @@ -0,0 +1,12 @@ +namespace ContinuousTests +{ + public class TestFactory + { + public ContinuousTest[] CreateTests() + { + var types = GetType().Assembly.GetTypes(); + var testTypes = types.Where(t => typeof(ContinuousTest).IsAssignableFrom(t) && !t.IsAbstract); + return testTypes.Select(t => (ContinuousTest)Activator.CreateInstance(t)!).ToArray(); + } + } +} diff --git a/ContinuousTests/TestFinder.cs b/ContinuousTests/TestFinder.cs deleted file mode 100644 index 349e9572..00000000 --- a/ContinuousTests/TestFinder.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace ContinuousTests -{ - public class TestFinder - { - private readonly List testList = new List(); - - public IContinuousTest[] GetTests() - { - if (!testList.Any()) FindTests(); - return testList.ToArray(); - } - - private void FindTests() - { - var types = GetType().Assembly.GetTypes(); - var testTypes = types.Where(t => typeof(IContinuousTest).IsAssignableFrom(t) && !t.IsAbstract); - foreach (var testType in testTypes) - { - var t = Activator.CreateInstance(testType); - testList.Add((IContinuousTest)t!); - } - } - } -} diff --git a/ContinuousTests/TestRun.cs b/ContinuousTests/TestRun.cs index 91387443..43e71b0c 100644 --- a/ContinuousTests/TestRun.cs +++ b/ContinuousTests/TestRun.cs @@ -7,29 +7,30 @@ namespace ContinuousTests public class TestRun { private readonly Random random = new Random(); + private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory(); private readonly Configuration config; private readonly BaseLog log; - private readonly TestFinder testFinder; - private readonly CodexNode[] nodes; + private readonly TestFactory testFinder; private readonly FileManager fileManager; + private ITimeSet timeSet; - public TestRun(Configuration config, BaseLog log, TestFinder testFinder, CodexNode[] nodes) + public TestRun(Configuration config, BaseLog log, TestFactory testFinder) { this.config = config; this.log = log; this.testFinder = testFinder; - this.nodes = nodes; fileManager = new FileManager(log, new DistTestCore.Configuration()); + timeSet = new DefaultTimeSet(); } public void Run() { - var remainingTests = testFinder.GetTests().ToList(); + var remainingTests = testFinder.CreateTests().ToList(); while (remainingTests.Any()) { var test = PickOneRandom(remainingTests); - var selectedNodes = SelectRandomNodes(test.RequiredNumberOfNodes); - AssignEssentials(test, selectedNodes); + var nodes = CreateRandomNodes(test.RequiredNumberOfNodes); + AssignEssentials(test, nodes); fileManager.PushFileSet(); log.Log($"Start '{test.Name}'"); @@ -49,33 +50,34 @@ namespace ContinuousTests } } - private void AssignEssentials(IContinuousTest test, CodexNode[] nodes) + private void AssignEssentials(ContinuousTest test, CodexNode[] nodes) { - var t = (ContinuousTest)test; - t.Nodes = nodes; - t.Log = log; - t.FileManager = fileManager; + test.Initialize(nodes, log, fileManager); } - private void ClearEssentials(IContinuousTest test) + private void ClearEssentials(ContinuousTest test) { - var t = (ContinuousTest)test; - t.Nodes = null!; - t.Log = null!; - t.FileManager = null!; + // Looks a little strange, but prevents finished test from interacting further. + test.Initialize(null!, null!, null!); } - private CodexNode[] SelectRandomNodes(int number) + private string[] SelectRandomUrls(int number) { - var remainingNodes = nodes.ToList(); - var result = new CodexNode[number]; + var urls = config.CodexUrls.ToList(); + var result = new string[number]; for (var i = 0; i < number; i++) { - result[i] = PickOneRandom(remainingNodes); + result[i] = PickOneRandom(urls); } return result; } + private CodexNode[] CreateRandomNodes(int number) + { + var urls = SelectRandomUrls(number); + return codexNodeFactory.Create(urls, log, timeSet); + } + private T PickOneRandom(List remainingItems) { var i = random.Next(0, remainingItems.Count); diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs new file mode 100644 index 00000000..c38bc922 --- /dev/null +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -0,0 +1,22 @@ +using DistTestCore; +using NUnit.Framework; + +namespace ContinuousTests.Tests +{ + public class TwoClientTest : ContinuousTest + { + public override int RequiredNumberOfNodes => 2; + + public override void Run() + { + var file = FileManager.GenerateTestFile(10.MB()); + + var cid = UploadFile(Nodes[0], file); + Assert.That(cid, Is.Not.Null); + + var dl = DownloadContent(Nodes[1], cid!); + + dl.AssertIsEqual(file); + } + } +}