diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 3e76298..edfc05b 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -1,6 +1,7 @@ using DistTestCore; using DistTestCore.Codex; using DistTestCore.Logs; +using FileUtils; using KubernetesWorkflow; using Logging; diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 1e2b3a4..ef95fb9 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -6,6 +6,7 @@ using KubernetesWorkflow; using NUnit.Framework.Internal; using System.Reflection; using static Program; +using FileUtils; namespace ContinuousTests { @@ -38,7 +39,7 @@ namespace ContinuousTests nodes = CreateRandomNodes(); dataFolder = config.DataPath + "-" + Guid.NewGuid(); - fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration()); + fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration().GetFileManagerFolder()); } public void Run(EventWaitHandle runFinishedHandle) diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs index 0ec268f..b2dd5d3 100644 --- a/ContinuousTests/Tests/HoldMyBeerTest.cs +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -1,4 +1,5 @@ using DistTestCore; +using FileUtils; using NUnit.Framework; using Utils; diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index e179944..5e2b841 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -1,4 +1,5 @@ using DistTestCore; +using FileUtils; using NUnit.Framework; using Utils; diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 155a24f..5439b5a 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -3,6 +3,7 @@ using DistTestCore.Helpers; using DistTestCore.Logs; using DistTestCore.Marketplace; using DistTestCore.Metrics; +using FileUtils; using KubernetesWorkflow; using Logging; using NUnit.Framework; @@ -100,9 +101,7 @@ namespace DistTestCore /// public void ScopedTestFiles(Action action) { - Get().FileManager.PushFileSet(); - action(); - Get().FileManager.PopFileSet(); + Get().FileManager.ScopedFiles(action); } public IOnlineCodexNode SetupCodexBootstrapNode() diff --git a/DistTestCore/DistTestCore.csproj b/DistTestCore/DistTestCore.csproj index 94a2271..565e175 100644 --- a/DistTestCore/DistTestCore.csproj +++ b/DistTestCore/DistTestCore.csproj @@ -27,6 +27,7 @@ + diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index c2af415..d541194 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using FileUtils; using Logging; using Utils; using static DistTestCore.Helpers.FullConnectivityHelper; @@ -43,7 +44,11 @@ namespace DistTestCore.Helpers public PeerConnectionState Check(Entry from, Entry to) { - fileManager.PushFileSet(); + return fileManager.ScopedFiles(() => CheckConnectivity(from, to)); + } + + private PeerConnectionState CheckConnectivity(Entry from, Entry to) + { var expectedFile = GenerateTestFile(from.Node, to.Node); using var uploadStream = File.OpenRead(expectedFile.Filename); @@ -61,11 +66,6 @@ namespace DistTestCore.Helpers // We consider that as no-connection for the purpose of this test. return PeerConnectionState.NoConnection; } - finally - { - fileManager.PopFileSet(); - } - // Should an exception occur during upload, then this try is inconclusive and we try again next loop. } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 52fb81f..2d1ee9d 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -2,6 +2,7 @@ using DistTestCore.Logs; using DistTestCore.Marketplace; using DistTestCore.Metrics; +using FileUtils; using Logging; using NUnit.Framework; using Utils; diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index d3f3f0f..0a364c8 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -2,6 +2,7 @@ using DistTestCore.Logs; using DistTestCore.Marketplace; using DistTestCore.Metrics; +using FileUtils; using KubernetesWorkflow; using Logging; using Utils; @@ -20,7 +21,7 @@ namespace DistTestCore WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace); - FileManager = new FileManager(Log, configuration); + FileManager = new FileManager(Log, configuration.GetFileManagerFolder()); CodexStarter = new CodexStarter(this); PrometheusStarter = new PrometheusStarter(this); GrafanaStarter = new GrafanaStarter(this); diff --git a/DistTestCore/FileManager.cs b/FileUtils/FileManager.cs similarity index 58% rename from DistTestCore/FileManager.cs rename to FileUtils/FileManager.cs index ab86473..58bb0a8 100644 --- a/DistTestCore/FileManager.cs +++ b/FileUtils/FileManager.cs @@ -2,15 +2,15 @@ using NUnit.Framework; using Utils; -namespace DistTestCore +namespace FileUtils { public interface IFileManager { TestFile CreateEmptyTestFile(string label = ""); TestFile GenerateTestFile(ByteSize size, string label = ""); void DeleteAllTestFiles(); - void PushFileSet(); - void PopFileSet(); + void ScopedFiles(Action action); + T ScopedFiles(Func action); } public class FileManager : IFileManager @@ -22,9 +22,9 @@ namespace DistTestCore private readonly string folder; private readonly List> fileSetStack = new List>(); - public FileManager(BaseLog log, Configuration configuration) + public FileManager(BaseLog log, string rootFolder) { - folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5")); + folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5")); EnsureDirectory(); this.log = log; @@ -52,12 +52,27 @@ namespace DistTestCore DeleteDirectory(); } - public void PushFileSet() + public void ScopedFiles(Action action) + { + PushFileSet(); + action(); + PopFileSet(); + } + + public T ScopedFiles(Func action) + { + PushFileSet(); + var result = action(); + PopFileSet(); + return result; + } + + private void PushFileSet() { fileSetStack.Add(new List()); } - public void PopFileSet() + private void PopFileSet() { if (!fileSetStack.Any()) return; var pop = fileSetStack.Last(); @@ -138,81 +153,4 @@ namespace DistTestCore Directory.Delete(folder, true); } } - - public class TestFile - { - private readonly BaseLog log; - - public TestFile(BaseLog log, string filename, string label) - { - this.log = log; - Filename = filename; - Label = label; - } - - public string Filename { get; } - public string Label { get; } - - public void AssertIsEqual(TestFile? actual) - { - var sw = Stopwatch.Begin(log); - try - { - AssertEqual(actual); - } - finally - { - sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}"); - } - } - - public string Describe() - { - var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})"; - if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix; - return $"'{Filename}'{sizePostfix}"; - } - - private void AssertEqual(TestFile? actual) - { - if (actual == null) Assert.Fail("TestFile is null."); - if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); - - Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); - - using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); - using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); - - var bytesExpected = new byte[FileManager.ChunkSize]; - var bytesActual = new byte[FileManager.ChunkSize]; - - var readExpected = 0; - var readActual = 0; - - while (true) - { - readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); - readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); - - if (readExpected == 0 && readActual == 0) - { - log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'."); - return; - } - - Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); - - for (var i = 0; i < readActual; i++) - { - if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal."); - } - } - } - - private long GetFileSize() - { - var info = new FileInfo(Filename); - return info.Length; - } - } } diff --git a/FileUtils/FileUtils.csproj b/FileUtils/FileUtils.csproj new file mode 100644 index 0000000..10c714c --- /dev/null +++ b/FileUtils/FileUtils.csproj @@ -0,0 +1,14 @@ + + + + net7.0 + enable + enable + + + + + + + + diff --git a/FileUtils/TestFile.cs b/FileUtils/TestFile.cs new file mode 100644 index 0000000..3f22926 --- /dev/null +++ b/FileUtils/TestFile.cs @@ -0,0 +1,83 @@ +using Logging; +using NUnit.Framework; +using Utils; + +namespace FileUtils +{ + public class TestFile + { + private readonly BaseLog log; + + public TestFile(BaseLog log, string filename, string label) + { + this.log = log; + Filename = filename; + Label = label; + } + + public string Filename { get; } + public string Label { get; } + + public void AssertIsEqual(TestFile? actual) + { + var sw = Stopwatch.Begin(log); + try + { + AssertEqual(actual); + } + finally + { + sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}"); + } + } + + public string Describe() + { + var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})"; + if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix; + return $"'{Filename}'{sizePostfix}"; + } + + private void AssertEqual(TestFile? actual) + { + if (actual == null) Assert.Fail("TestFile is null."); + if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + + Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); + + using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); + using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); + + var bytesExpected = new byte[FileManager.ChunkSize]; + var bytesActual = new byte[FileManager.ChunkSize]; + + var readExpected = 0; + var readActual = 0; + + while (true) + { + readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); + readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); + + if (readExpected == 0 && readActual == 0) + { + log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'."); + return; + } + + Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); + + for (var i = 0; i < readActual; i++) + { + if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal."); + } + } + } + + private long GetFileSize() + { + var info = new FileInfo(Filename); + return info.Length; + } + } +} diff --git a/LongTests/BasicTests/DownloadTests.cs b/LongTests/BasicTests/DownloadTests.cs index 5e01e3c..0cf97e9 100644 --- a/LongTests/BasicTests/DownloadTests.cs +++ b/LongTests/BasicTests/DownloadTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using FileUtils; using NUnit.Framework; using Utils; diff --git a/LongTests/BasicTests/UploadTests.cs b/LongTests/BasicTests/UploadTests.cs index 69823eb..516586e 100644 --- a/LongTests/BasicTests/UploadTests.cs +++ b/LongTests/BasicTests/UploadTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using FileUtils; using NUnit.Framework; using Utils; diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 1492ce1..76b37ab 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -10,7 +10,7 @@ namespace Tests.BasicTests [Test] public void CodexLogExample() { - var primary = SetupCodexNode(); + var primary = SetupCodexNodes(2)[0]; primary.UploadFile(GenerateTestFile(5.MB())); diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 8e696b3..b080268 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -23,7 +23,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDeployer", "CodexNe EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ArgsUniform", "ArgsUniform\ArgsUniform.csproj", "{634324B1-E359-42B4-A269-BDC429936B3C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexNetDownloader", "CodexNetDownloader\CodexNetDownloader.csproj", "{6CDF35D2-906A-4285-8E1F-4794588B948B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FileUtils", "FileUtils\FileUtils.csproj", "{ECC954DA-8D4E-49EE-83AD-80085A43DEEB}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -75,6 +77,10 @@ Global {6CDF35D2-906A-4285-8E1F-4794588B948B}.Debug|Any CPU.Build.0 = Debug|Any CPU {6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.ActiveCfg = Release|Any CPU {6CDF35D2-906A-4285-8E1F-4794588B948B}.Release|Any CPU.Build.0 = Release|Any CPU + {ECC954DA-8D4E-49EE-83AD-80085A43DEEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ECC954DA-8D4E-49EE-83AD-80085A43DEEB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ECC954DA-8D4E-49EE-83AD-80085A43DEEB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ECC954DA-8D4E-49EE-83AD-80085A43DEEB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE